MSC2530: added the ability to send media with captions (#3226)

Now that there is some support for [MSC2530](matrix-org/matrix-spec-proposals#2530), I gave adding sending captions a try. ( This is my first time with Rust 😄  )

I tried it on Element X with a hardcoded caption and it seems to work well
![image](https://github.com/matrix-org/matrix-rust-sdk/assets/683652/597e5ebf-f7f2-498f-97a4-ac98613c1134)

(It even got forwarded through mautrix-whatsapp and the caption was visible on the Whatsapp side)

---

* ffi: Expose filename and formatted body fields for media captions

In relevance to MSC2530

* MSC2530: added the ability to send media with captions

Signed-off-by: Marco Antonio Alvarez <surakin@gmail.com>

* signoff

Signed-off-by: Marco Antonio Alvarez <surakin@gmail.com>

* fixing the import messup

* fix missing parameters in documentation

* fix formatting

* move optional parameters to the end

* more formatting fixes

* more formatting fixes

* rename url parameter to filename in send_attachment and helpers

* fix send_attachment documentation example

* move caption and formatted_caption into attachmentconfig

* fix formatting

* fix formatting

* fix formatting (hopefully the last one)

* updated stale comments

* simplify attachment message comments

---------

Signed-off-by: Marco Antonio Alvarez <surakin@gmail.com>
Co-authored-by: SpiritCroc <dev@spiritcroc.de>

bindings/matrix-sdk-ffi/src/timeline/mod.rs

@@ -52,19 +52,21 @@ use tokio::{
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
+use self::content::{Reaction, ReactionSenderData, TimelineItemContent};
 use crate::{
     client::ProgressWatcher,
     error::{ClientError, RoomError},
     helpers::unwrap_or_clone_arc,
-    ruma::{AssetType, AudioInfo, FileInfo, ImageInfo, PollKind, ThumbnailInfo, VideoInfo},
+    ruma::{
+        AssetType, AudioInfo, FileInfo, FormattedBody, ImageInfo, PollKind, ThumbnailInfo,
+        VideoInfo,
+    },
     task_handle::TaskHandle,
     RUNTIME,
 };
 
 mod content;
 
-pub use self::content::{Reaction, ReactionSenderData, TimelineItemContent};
-
 #[derive(uniffi::Object)]
 #[repr(transparent)]
 pub struct Timeline {
@@ -106,12 +108,12 @@ impl Timeline {
 
     async fn send_attachment(
         &self,
-        url: String,
+        filename: String,
         mime_type: Mime,
         attachment_config: AttachmentConfig,
         progress_watcher: Option<Box<dyn ProgressWatcher>>,
     ) -> Result<(), RoomError> {
-        let request = self.inner.send_attachment(url, mime_type, attachment_config);
+        let request = self.inner.send_attachment(filename, mime_type, attachment_config);
         if let Some(progress_watcher) = progress_watcher {
             let mut subscriber = request.subscribe_to_send_progress();
             RUNTIME.spawn(async move {
@@ -218,6 +220,8 @@ impl Timeline {
         url: String,
         thumbnail_url: Option<String>,
         image_info: ImageInfo,
+        caption: Option<String>,
+        formatted_caption: Option<FormattedBody>,
         progress_watcher: Option<Box<dyn ProgressWatcher>>,
     ) -> Arc<SendAttachmentJoinHandle> {
         SendAttachmentJoinHandle::new(RUNTIME.spawn(async move {
@@ -238,7 +242,9 @@ impl Timeline {
                     AttachmentConfig::with_thumbnail(thumbnail).info(attachment_info)
                 }
                 _ => AttachmentConfig::new().info(attachment_info),
-            };
+            }
+            .caption(caption)
+            .formatted_caption(formatted_caption.map(Into::into));
 
             self.send_attachment(url, mime_type, attachment_config, progress_watcher).await
         }))
@@ -249,6 +255,8 @@ impl Timeline {
         url: String,
         thumbnail_url: Option<String>,
         video_info: VideoInfo,
+        caption: Option<String>,
+        formatted_caption: Option<FormattedBody>,
         progress_watcher: Option<Box<dyn ProgressWatcher>>,
     ) -> Arc<SendAttachmentJoinHandle> {
         SendAttachmentJoinHandle::new(RUNTIME.spawn(async move {
@@ -269,7 +277,9 @@ impl Timeline {
                     AttachmentConfig::with_thumbnail(thumbnail).info(attachment_info)
                 }
                 _ => AttachmentConfig::new().info(attachment_info),
-            };
+            }
+            .caption(caption)
+            .formatted_caption(formatted_caption.map(Into::into));
 
             self.send_attachment(url, mime_type, attachment_config, progress_watcher).await
         }))
@@ -279,6 +289,8 @@ impl Timeline {
         self: Arc<Self>,
         url: String,
         audio_info: AudioInfo,
+        caption: Option<String>,
+        formatted_caption: Option<FormattedBody>,
         progress_watcher: Option<Box<dyn ProgressWatcher>>,
     ) -> Arc<SendAttachmentJoinHandle> {
         SendAttachmentJoinHandle::new(RUNTIME.spawn(async move {
@@ -291,7 +303,10 @@ impl Timeline {
                 .map_err(|_| RoomError::InvalidAttachmentData)?;
 
             let attachment_info = AttachmentInfo::Audio(base_audio_info);
-            let attachment_config = AttachmentConfig::new().info(attachment_info);
+            let attachment_config = AttachmentConfig::new()
+                .info(attachment_info)
+                .caption(caption)
+                .formatted_caption(formatted_caption.map(Into::into));
 
             self.send_attachment(url, mime_type, attachment_config, progress_watcher).await
         }))
@@ -302,6 +317,8 @@ impl Timeline {
         url: String,
         audio_info: AudioInfo,
         waveform: Vec<u16>,
+        caption: Option<String>,
+        formatted_caption: Option<FormattedBody>,
         progress_watcher: Option<Box<dyn ProgressWatcher>>,
     ) -> Arc<SendAttachmentJoinHandle> {
         SendAttachmentJoinHandle::new(RUNTIME.spawn(async move {
@@ -315,7 +332,10 @@ impl Timeline {
 
             let attachment_info =
                 AttachmentInfo::Voice { audio_info: base_audio_info, waveform: Some(waveform) };
-            let attachment_config = AttachmentConfig::new().info(attachment_info);
+            let attachment_config = AttachmentConfig::new()
+                .info(attachment_info)
+                .caption(caption)
+                .formatted_caption(formatted_caption.map(Into::into));
 
             self.send_attachment(url, mime_type, attachment_config, progress_watcher).await
         }))

crates/matrix-sdk-ui/src/timeline/futures.rs

@@ -10,7 +10,7 @@ use super::{Error, Timeline};
 
 pub struct SendAttachment<'a> {
     timeline: &'a Timeline,
-    url: String,
+    filename: String,
     mime_type: Mime,
     config: AttachmentConfig,
     tracing_span: Span,
@@ -20,13 +20,13 @@ pub struct SendAttachment<'a> {
 impl<'a> SendAttachment<'a> {
     pub(crate) fn new(
         timeline: &'a Timeline,
-        url: String,
+        filename: String,
         mime_type: Mime,
         config: AttachmentConfig,
     ) -> Self {
         Self {
             timeline,
-            url,
+            filename,
             mime_type,
             config,
             tracing_span: Span::current(),
@@ -47,14 +47,14 @@ impl<'a> IntoFuture for SendAttachment<'a> {
     boxed_into_future!(extra_bounds: 'a);
 
     fn into_future(self) -> Self::IntoFuture {
-        let Self { timeline, url, mime_type, config, tracing_span, send_progress } = self;
+        let Self { timeline, filename, mime_type, config, tracing_span, send_progress } = self;
         let fut = async move {
-            let body = Path::new(&url)
+            let body = Path::new(&filename)
                 .file_name()
                 .ok_or(Error::InvalidAttachmentFileName)?
                 .to_str()
                 .expect("path was created from UTF-8 string, hence filename part is UTF-8 too");
-            let data = fs::read(&url).map_err(|_| Error::InvalidAttachmentData)?;
+            let data = fs::read(&filename).map_err(|_| Error::InvalidAttachmentData)?;
 
             timeline
                 .room()

crates/matrix-sdk-ui/src/timeline/mod.rs

@@ -513,21 +513,22 @@ impl Timeline {
     ///
     /// # Arguments
     ///
-    /// * `url` - The url for the file to be sent
+    /// * `filename` - The filename of the file to be sent
     ///
     /// * `mime_type` - The attachment's mime type
     ///
     /// * `config` - An attachment configuration object containing details about
     ///   the attachment
+    ///
     /// like a thumbnail, its size, duration etc.
     #[instrument(skip_all)]
     pub fn send_attachment(
         &self,
-        url: String,
+        filename: String,
         mime_type: Mime,
         config: AttachmentConfig,
     ) -> SendAttachment<'_> {
-        SendAttachment::new(self, url, mime_type, config)
+        SendAttachment::new(self, filename, mime_type, config)
     }
 
     /// Retry sending a message that previously failed to send.

crates/matrix-sdk/src/attachment.rs

@@ -23,7 +23,7 @@ use image::GenericImageView;
 use ruma::{
     assign,
     events::room::{
-        message::{AudioInfo, FileInfo, VideoInfo},
+        message::{AudioInfo, FileInfo, FormattedBody, VideoInfo},
         ImageInfo, ThumbnailInfo,
     },
     OwnedTransactionId, TransactionId, UInt,
@@ -190,6 +190,8 @@ pub struct AttachmentConfig {
     pub(crate) txn_id: Option<OwnedTransactionId>,
     pub(crate) info: Option<AttachmentInfo>,
     pub(crate) thumbnail: Option<Thumbnail>,
+    pub(crate) caption: Option<String>,
+    pub(crate) formatted_caption: Option<FormattedBody>,
     #[cfg(feature = "image-proc")]
     pub(crate) generate_thumbnail: bool,
     #[cfg(feature = "image-proc")]
@@ -205,6 +207,8 @@ impl AttachmentConfig {
             txn_id: Default::default(),
             info: Default::default(),
             thumbnail: None,
+            caption: None,
+            formatted_caption: None,
             #[cfg(feature = "image-proc")]
             generate_thumbnail: Default::default(),
             #[cfg(feature = "image-proc")]
@@ -247,6 +251,8 @@ impl AttachmentConfig {
             txn_id: Default::default(),
             info: Default::default(),
             thumbnail: Some(thumbnail),
+            caption: None,
+            formatted_caption: None,
             #[cfg(feature = "image-proc")]
             generate_thumbnail: Default::default(),
             #[cfg(feature = "image-proc")]
@@ -278,6 +284,26 @@ impl AttachmentConfig {
         self.info = Some(info);
         self
     }
+
+    /// Set the optional caption
+    ///
+    /// # Arguments
+    ///
+    /// * `caption` - The optional caption
+    pub fn caption(mut self, caption: Option<String>) -> Self {
+        self.caption = caption;
+        self
+    }
+
+    /// Set the optional formatted caption
+    ///
+    /// # Arguments
+    ///
+    /// * `formatted_caption` - The optional formatted caption
+    pub fn formatted_caption(mut self, formatted_caption: Option<FormattedBody>) -> Self {
+        self.formatted_caption = formatted_caption;
+        self
+    }
 }
 
 impl Default for AttachmentConfig {
@@ -331,7 +357,7 @@ impl Default for AttachmentConfig {
 ///
 /// if let Some(room) = client.get_room(&room_id) {
 ///     room.send_attachment(
-///         "My favorite cat",
+///         "my_favorite_cat.jpg",
 ///         &mime::IMAGE_JPEG,
 ///         image,
 ///         config,

crates/matrix-sdk/src/encryption/mod.rs

@@ -68,7 +68,7 @@ use self::{
     tasks::{BackupDownloadTask, BackupUploadingTask, ClientTasks},
 };
 use crate::{
-    attachment::{AttachmentInfo, Thumbnail},
+    attachment::{AttachmentConfig, Thumbnail},
     client::ClientInner,
     encryption::{
         identities::{Device, UserDevices},
@@ -298,18 +298,17 @@ impl Client {
     }
 
     /// Encrypt and upload the file to be read from `reader` and construct an
-    /// attachment message with `body`, `content_type`, `info` and `thumbnail`.
+    /// attachment message.
     pub(crate) async fn prepare_encrypted_attachment_message(
         &self,
-        body: &str,
+        filename: &str,
         content_type: &mime::Mime,
         data: Vec<u8>,
-        info: Option<AttachmentInfo>,
-        thumbnail: Option<Thumbnail>,
+        config: AttachmentConfig,
         send_progress: SharedObservable<TransmissionProgress>,
     ) -> Result<MessageType> {
         let upload_thumbnail =
-            self.upload_encrypted_thumbnail(thumbnail, content_type, send_progress.clone());
+            self.upload_encrypted_thumbnail(config.thumbnail, content_type, send_progress.clone());
 
         let upload_attachment = async {
             let mut cursor = Cursor::new(data);
@@ -321,15 +320,25 @@ impl Client {
         let ((thumbnail_source, thumbnail_info), file) =
             try_join(upload_thumbnail, upload_attachment).await?;
 
+        // if config.caption is set, use it as body, and filename as the file name
+        // otherwise, body is the filename, and the filename is not set
+        // https://github.com/tulir/matrix-spec-proposals/blob/body-as-caption/proposals/2530-body-as-caption.md
+        let (body, filename) = match config.caption {
+            Some(caption) => (caption, Some(filename.to_owned())),
+            None => (filename.to_owned(), None),
+        };
+
         Ok(match content_type.type_() {
             mime::IMAGE => {
-                let info = assign!(info.map(ImageInfo::from).unwrap_or_default(), {
+                let info = assign!(config.info.map(ImageInfo::from).unwrap_or_default(), {
                     mimetype: Some(content_type.as_ref().to_owned()),
                     thumbnail_source,
                     thumbnail_info
                 });
                 let content = assign!(ImageMessageEventContent::encrypted(body.to_owned(), file), {
-                    info: Some(Box::new(info))
+                    info: Some(Box::new(info)),
+                    formatted: config.formatted_caption,
+                    filename
                 });
                 MessageType::Image(content)
             }
@@ -339,22 +348,24 @@ impl Client {
                 MessageType::Audio(crate::media::update_audio_message_event(
                     audio_message_event_content,
                     content_type,
-                    info,
+                    config.info,
                 ))
             }
             mime::VIDEO => {
-                let info = assign!(info.map(VideoInfo::from).unwrap_or_default(), {
+                let info = assign!(config.info.map(VideoInfo::from).unwrap_or_default(), {
                     mimetype: Some(content_type.as_ref().to_owned()),
                     thumbnail_source,
                     thumbnail_info
                 });
                 let content = assign!(VideoMessageEventContent::encrypted(body.to_owned(), file), {
-                    info: Some(Box::new(info))
+                    info: Some(Box::new(info)),
+                    formatted: config.formatted_caption,
+                    filename
                 });
                 MessageType::Video(content)
             }
             _ => {
-                let info = assign!(info.map(FileInfo::from).unwrap_or_default(), {
+                let info = assign!(config.info.map(FileInfo::from).unwrap_or_default(), {
                     mimetype: Some(content_type.as_ref().to_owned()),
                     thumbnail_source,
                     thumbnail_info