diff --git a/bindings/matrix-sdk-ffi/src/event.rs b/bindings/matrix-sdk-ffi/src/event.rs index d2b0c831648..f988751d45f 100644 --- a/bindings/matrix-sdk-ffi/src/event.rs +++ b/bindings/matrix-sdk-ffi/src/event.rs @@ -673,6 +673,7 @@ impl From for ruma::events::MessageLikeEventType { #[derive(Debug, PartialEq, Clone, uniffi::Enum)] pub enum RoomMessageEventMessageType { Audio, + Custom, Emote, File, #[cfg(feature = "unstable-msc4274")] @@ -691,6 +692,7 @@ impl From for RoomMessageEventMessageType { fn from(val: ruma::events::room::message::MessageType) -> Self { match val { RumaMessageType::Audio { .. } => Self::Audio, + RumaMessageType::_Custom(_) => Self::Custom, RumaMessageType::Emote { .. } => Self::Emote, RumaMessageType::File { .. } => Self::File, #[cfg(feature = "unstable-msc4274")] @@ -741,3 +743,29 @@ impl TryFrom for TimelineEventItemId { } } } + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn room_message_event_message_type_marks_custom_messages_as_custom() { + let custom_message = RumaMessageType::new( + "org.example.letter", + "Dear Alice".to_owned(), + serde_json::from_value(json!({ + "letter_id": "lt-123", + "version": 1, + })) + .unwrap(), + ) + .unwrap(); + + assert_eq!( + RoomMessageEventMessageType::from(custom_message), + RoomMessageEventMessageType::Custom + ); + } +} diff --git a/bindings/matrix-sdk-ffi/src/room/mod.rs b/bindings/matrix-sdk-ffi/src/room/mod.rs index ed6b456d537..7715427a2d3 100644 --- a/bindings/matrix-sdk-ffi/src/room/mod.rs +++ b/bindings/matrix-sdk-ffi/src/room/mod.rs @@ -21,6 +21,7 @@ use matrix_sdk::{ DraftAttachment as SdkDraftAttachment, DraftAttachmentContent, DraftThumbnail, EncryptionState, PredecessorRoom as SdkPredecessorRoom, RoomHero as SdkRoomHero, RoomMemberships, RoomState, SuccessorRoom as SdkSuccessorRoom, + attachment::{BaseAudioInfo, BaseFileInfo, BaseImageInfo, BaseVideoInfo, Thumbnail as SdkThumbnail}, encryption::LocalTrust, room::{ Room as SdkRoom, RoomMemberRole, edit::EditedContent, power_levels::RoomPowerLevelChanges, @@ -62,7 +63,10 @@ use crate::{ live_locations_observer::LiveLocationsObserver, room_member::{RoomMember, RoomMemberWithSenderInfo}, room_preview::RoomPreview, - ruma::{AudioInfo, FileInfo, ImageInfo, MediaSource, ThumbnailInfo, VideoInfo}, + ruma::{ + AudioInfo, AudioMessageContent, FileInfo, FileMessageContent, ImageInfo, + ImageMessageContent, MediaSource, ThumbnailInfo, VideoInfo, VideoMessageContent, + }, runtime::get_runtime_handle, timeline::{ AbstractProgress, LatestEventValue, ReceiptType, SendHandle, Timeline, UploadSource, @@ -106,6 +110,32 @@ impl Room { pub(crate) fn new(inner: SdkRoom, utd_hook_manager: Option>) -> Self { Room { inner, utd_hook_manager } } + + async fn upload_attachment_source( + &self, + source: UploadSource, + mime_type: Option, + thumbnail: Option, + ) -> Result<(String, Arc, Option>), ClientError> { + let mime_str = mime_type + .as_ref() + .ok_or(RoomError::InvalidAttachmentMimeType) + .map_err(ClientError::from)?; + let mime_type = mime_str.parse::().map_err(|_| RoomError::InvalidAttachmentMimeType)?; + + let (data, filename) = read_upload_source(source)?; + let uploaded = self.inner.upload_attachment(&mime_type, data, thumbnail).await?; + + let source = Arc::new(uploaded.source.try_into()?); + let thumbnail_source = uploaded + .thumbnail + .as_ref() + .map(|(source, _)| MediaSource::try_from(source)) + .transpose()? + .map(Arc::new); + + Ok((filename, source, thumbnail_source)) + } } #[matrix_sdk_ffi_macros::export] @@ -438,6 +468,96 @@ impl Room { Ok(()) } + /// Upload image media for later use in a custom event without sending a media event. + pub async fn upload_image( + &self, + source: UploadSource, + thumbnail_source: Option, + mut image_info: ImageInfo, + ) -> Result { + BaseImageInfo::try_from(&image_info).map_err(|_| RoomError::InvalidAttachmentData)?; + + let thumbnail = build_upload_thumbnail(image_info.thumbnail_info.clone(), thumbnail_source)?; + let (filename, source, uploaded_thumbnail_source) = + self.upload_attachment_source(source, image_info.mimetype.clone(), thumbnail).await?; + + image_info.thumbnail_source = uploaded_thumbnail_source; + + Ok(ImageMessageContent { + filename, + caption: None, + formatted_caption: None, + source, + info: Some(image_info), + }) + } + + /// Upload video media for later use in a custom event without sending a media event. + pub async fn upload_video( + &self, + source: UploadSource, + thumbnail_source: Option, + mut video_info: VideoInfo, + ) -> Result { + BaseVideoInfo::try_from(&video_info).map_err(|_| RoomError::InvalidAttachmentData)?; + + let thumbnail = build_upload_thumbnail(video_info.thumbnail_info.clone(), thumbnail_source)?; + let (filename, source, uploaded_thumbnail_source) = + self.upload_attachment_source(source, video_info.mimetype.clone(), thumbnail).await?; + + video_info.thumbnail_source = uploaded_thumbnail_source; + + Ok(VideoMessageContent { + filename, + caption: None, + formatted_caption: None, + source, + info: Some(video_info), + }) + } + + /// Upload audio media for later use in a custom event without sending a media event. + pub async fn upload_audio( + &self, + source: UploadSource, + audio_info: AudioInfo, + ) -> Result { + BaseAudioInfo::try_from(&audio_info).map_err(|_| RoomError::InvalidAttachmentData)?; + + let (filename, source, _) = + self.upload_attachment_source(source, audio_info.mimetype.clone(), None).await?; + + Ok(AudioMessageContent { + filename, + caption: None, + formatted_caption: None, + source, + info: Some(audio_info), + audio: None, + voice: None, + }) + } + + /// Upload file media for later use in a custom event without sending a media event. + pub async fn upload_file( + &self, + source: UploadSource, + file_info: FileInfo, + ) -> Result { + BaseFileInfo::try_from(&file_info).map_err(|_| RoomError::InvalidAttachmentData)?; + + let (filename, source, _) = + self.upload_attachment_source(source, file_info.mimetype.clone(), None).await?; + + Ok(FileMessageContent { + filename, + caption: None, + formatted_caption: None, + source, + info: Some(file_info), + }) + } + /// Send a raw state event to the room. /// /// # Arguments @@ -1633,6 +1753,79 @@ fn read_upload_source(source: UploadSource) -> Result<(Vec, String), ClientE } } +fn build_upload_thumbnail( + thumbnail_info: Option, + thumbnail_source: Option, +) -> Result, ClientError> { + match (thumbnail_info, thumbnail_source) { + (Some(info), Some(source)) => { + let mime_type = info + .mimetype + .as_ref() + .ok_or(RoomError::InvalidThumbnailData) + .map_err(ClientError::from)? + .parse::() + .map_err(|_| RoomError::InvalidThumbnailData)?; + let height = u64_to_uint(info.height.ok_or(RoomError::InvalidThumbnailData)?); + let width = u64_to_uint(info.width.ok_or(RoomError::InvalidThumbnailData)?); + let size = u64_to_uint(info.size.ok_or(RoomError::InvalidThumbnailData)?); + let (data, _) = read_upload_source(source)?; + + Ok(Some(SdkThumbnail { data, content_type: mime_type, height, width, size })) + } + (None, None) => Ok(None), + _ => Err(ClientError::from(RoomError::InvalidThumbnailData)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn thumbnail_info() -> ThumbnailInfo { + ThumbnailInfo { + height: Some(60), + width: Some(80), + mimetype: Some("image/jpeg".to_owned()), + size: Some(128), + } + } + + fn thumbnail_source() -> UploadSource { + UploadSource::Data { + bytes: vec![1, 2, 3], + filename: "thumb.jpg".to_owned(), + } + } + + #[test] + fn build_upload_thumbnail_rejects_missing_thumbnail_source() { + let result = build_upload_thumbnail(Some(thumbnail_info()), None); + + assert!(matches!(result, Err(ClientError::Generic { msg, .. }) if msg.contains("Invalid thumbnail data"))); + } + + #[test] + fn build_upload_thumbnail_rejects_missing_thumbnail_info() { + let result = build_upload_thumbnail(None, Some(thumbnail_source())); + + assert!(matches!(result, Err(ClientError::Generic { msg, .. }) if msg.contains("Invalid thumbnail data"))); + } + + #[test] + fn build_upload_thumbnail_returns_thumbnail_when_inputs_are_complete() { + let result = build_upload_thumbnail(Some(thumbnail_info()), Some(thumbnail_source())) + .unwrap() + .unwrap(); + + assert_eq!(result.data, vec![1, 2, 3]); + assert_eq!(result.content_type, "image/jpeg".parse::().unwrap()); + assert_eq!(result.height, u64_to_uint(60)); + assert_eq!(result.width, u64_to_uint(80)); + assert_eq!(result.size, u64_to_uint(128)); + } +} + impl TryFrom for SdkDraftAttachment { type Error = ClientError; diff --git a/bindings/matrix-sdk-ffi/src/ruma.rs b/bindings/matrix-sdk-ffi/src/ruma.rs index 03393dfeec8..0c61448fbd5 100644 --- a/bindings/matrix-sdk-ffi/src/ruma.rs +++ b/bindings/matrix-sdk-ffi/src/ruma.rs @@ -374,6 +374,11 @@ pub enum MessageType { Location { content: LocationContent, }, + Custom { + msgtype: String, + body: String, + data_json: String, + }, Other { msgtype: String, body: String, @@ -422,6 +427,10 @@ impl TryFrom for RumaMessageType { MessageType::Location { content } => { Self::Location(RumaLocationMessageEventContent::new(content.body, content.geo_uri)) } + MessageType::Custom { msgtype, body, data_json } => { + let data: JsonObject = serde_json::from_str(&data_json)?; + Self::new(&msgtype, body, data)? + } MessageType::Other { msgtype, body } => { Self::new(&msgtype, body, JsonObject::default())? } @@ -471,6 +480,11 @@ impl TryFrom for MessageType { }, } } + RumaMessageType::_Custom(_) => MessageType::Custom { + msgtype: value.msgtype().to_owned(), + body: value.body().to_owned(), + data_json: serde_json::to_string(&value.data())?, + }, _ => MessageType::Other { msgtype: value.msgtype().to_owned(), body: value.body().to_owned(), @@ -479,6 +493,65 @@ impl TryFrom for MessageType { } } +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn custom_message_type_from_ruma_preserves_data_json() { + let data = serde_json::from_value(json!({ + "org.example.field": "value", + "count": 7, + })) + .unwrap(); + let ruma_message_type = + RumaMessageType::new("org.example.custom", "Fallback body".to_owned(), data).unwrap(); + + let message_type = MessageType::try_from(ruma_message_type).unwrap(); + + let MessageType::Custom { msgtype, body, data_json } = message_type else { + panic!("expected custom message type"); + }; + + assert_eq!(msgtype, "org.example.custom"); + assert_eq!(body, "Fallback body"); + assert_eq!( + serde_json::from_str::(&data_json).unwrap(), + json!({ + "org.example.field": "value", + "count": 7, + }) + ); + } + + #[test] + fn custom_message_type_to_ruma_restores_data_json() { + let message_type = MessageType::Custom { + msgtype: "org.example.custom".to_owned(), + body: "Fallback body".to_owned(), + data_json: json!({ + "org.example.field": "value", + "count": 7, + }) + .to_string(), + }; + + let ruma_message_type = RumaMessageType::try_from(message_type).unwrap(); + + assert_eq!(ruma_message_type.msgtype(), "org.example.custom"); + assert_eq!(ruma_message_type.body(), "Fallback body"); + assert_eq!( + serde_json::to_value(ruma_message_type.data()).unwrap(), + json!({ + "org.example.field": "value", + "count": 7, + }) + ); + } +} + #[derive(Clone, uniffi::Enum)] pub enum RtcNotificationType { Ring, diff --git a/crates/matrix-sdk-ui/src/timeline/controller/mod.rs b/crates/matrix-sdk-ui/src/timeline/controller/mod.rs index 554ac3612a8..b2a5d75dcd9 100644 --- a/crates/matrix-sdk-ui/src/timeline/controller/mod.rs +++ b/crates/matrix-sdk-ui/src/timeline/controller/mod.rs @@ -296,6 +296,7 @@ pub fn default_event_filter(event: &AnySyncTimelineEvent, rules: &RoomVersionRul | MessageType::Text(_) | MessageType::Video(_) | MessageType::VerificationRequest(_) => true, + _ if content.msgtype.msgtype() == "org.letro.letter" => true, #[cfg(feature = "unstable-msc4274")] MessageType::Gallery(_) => true, _ => false, diff --git a/crates/matrix-sdk/src/attachment.rs b/crates/matrix-sdk/src/attachment.rs index 67c2e0fe817..bf29c2ad1ed 100644 --- a/crates/matrix-sdk/src/attachment.rs +++ b/crates/matrix-sdk/src/attachment.rs @@ -21,7 +21,7 @@ use ruma::{ events::{ Mentions, room::{ - ImageInfo, ThumbnailInfo, + ImageInfo, MediaSource, ThumbnailInfo, message::{AudioInfo, FileInfo, TextMessageEventContent, VideoInfo}, }, }, @@ -177,6 +177,16 @@ impl Thumbnail { } } +/// The uploaded sources for an attachment and its optional thumbnail. +#[derive(Debug, Clone)] +pub struct UploadedAttachment { + /// The uploaded source for the primary attachment. + pub source: MediaSource, + + /// The uploaded source and info for the thumbnail, if any. + pub thumbnail: Option<(MediaSource, Box)>, +} + /// Configuration for sending an attachment. #[derive(Debug, Default)] pub struct AttachmentConfig { diff --git a/crates/matrix-sdk/src/room/mod.rs b/crates/matrix-sdk/src/room/mod.rs index db444855762..86215d03106 100644 --- a/crates/matrix-sdk/src/room/mod.rs +++ b/crates/matrix-sdk/src/room/mod.rs @@ -170,7 +170,7 @@ use crate::event_cache::EventCache; use crate::room::futures::{SendRawStateEvent, SendStateEvent}; use crate::{ BaseRoom, Client, Error, HttpResult, Result, RoomState, TransmissionProgress, - attachment::{AttachmentConfig, AttachmentInfo}, + attachment::{AttachmentConfig, AttachmentInfo, UploadedAttachment}, client::WeakClient, config::RequestConfig, error::{BeaconError, WrongRoomState}, @@ -2683,6 +2683,27 @@ impl Room { SendAttachment::new(self, filename.into(), content_type, data, config) } + /// Upload an attachment to this room without sending a media event. + /// + /// This uses the same encrypted/plain upload logic as [`Self::send_attachment`] + /// but stops after the media and optional thumbnail have been uploaded. + #[instrument(skip_all)] + pub async fn upload_attachment( + &self, + content_type: &Mime, + data: Vec, + thumbnail: Option, + ) -> Result { + self.prepare_attachment_upload( + content_type, + data, + thumbnail, + SharedObservable::new(TransmissionProgress::default()), + false, + ) + .await + } + /// Prepare and send an attachment to this room. /// /// This will upload the given data that the reader produces using the @@ -2720,12 +2741,51 @@ impl Room { send_progress: SharedObservable, store_in_cache: bool, ) -> Result { - self.ensure_room_joined()?; - let txn_id = config.txn_id.take(); let mentions = config.mentions.take(); + let UploadedAttachment { source: media_source, thumbnail } = self + .prepare_attachment_upload( + content_type, + data, + config.thumbnail.take(), + send_progress, + store_in_cache, + ) + .await?; + + let content = self + .make_media_event( + Room::make_attachment_type( + content_type, + filename, + media_source, + config.caption, + config.info, + thumbnail, + ), + mentions, + config.reply, + ) + .await?; + + let mut fut = self.send(content); + if let Some(txn_id) = txn_id { + fut = fut.with_transaction_id(txn_id); + } + + fut.await.map(|result| result.response) + } - let thumbnail = config.thumbnail.take(); + #[instrument(skip_all)] + async fn prepare_attachment_upload( + &self, + content_type: &Mime, + data: Vec, + thumbnail: Option, + send_progress: SharedObservable, + store_in_cache: bool, + ) -> Result { + self.ensure_room_joined()?; // If necessary, store caching data for the thumbnail ahead of time. let thumbnail_cache_info = if store_in_cache { @@ -2737,7 +2797,7 @@ impl Room { }; #[cfg(feature = "e2e-encryption")] - let (media_source, thumbnail) = if self.latest_encryption_state().await?.is_encrypted() { + let (source, thumbnail) = if self.latest_encryption_state().await?.is_encrypted() { self.client .upload_encrypted_media_and_thumbnail(&data, thumbnail, send_progress) .await? @@ -2756,7 +2816,7 @@ impl Room { }; #[cfg(not(feature = "e2e-encryption"))] - let (media_source, thumbnail) = self + let (source, thumbnail) = self .client .media() .upload_plain_media_and_thumbnail(content_type, data.clone(), thumbnail, send_progress) @@ -2769,8 +2829,7 @@ impl Room { // properly, so only log errors during caching. debug!("caching the media"); - let request = - MediaRequestParameters { source: media_source.clone(), format: MediaFormat::File }; + let request = MediaRequestParameters { source: source.clone(), format: MediaFormat::File }; if let Err(err) = media_store_lock_guard .add_media_content(&request, data, IgnoreMediaRetentionPolicy::No) @@ -2779,13 +2838,13 @@ impl Room { warn!("unable to cache the media after uploading it: {err}"); } - if let Some(((data, height, width), source)) = + if let Some(((data, height, width), thumbnail_source)) = thumbnail_cache_info.zip(thumbnail.as_ref().map(|tuple| &tuple.0)) { debug!("caching the thumbnail"); let request = MediaRequestParameters { - source: source.clone(), + source: thumbnail_source.clone(), format: MediaFormat::Thumbnail(MediaThumbnailSettings::new(width, height)), }; @@ -2798,27 +2857,7 @@ impl Room { } } - let content = self - .make_media_event( - Room::make_attachment_type( - content_type, - filename, - media_source, - config.caption, - config.info, - thumbnail, - ), - mentions, - config.reply, - ) - .await?; - - let mut fut = self.send(content); - if let Some(txn_id) = txn_id { - fut = fut.with_transaction_id(txn_id); - } - - fut.await.map(|result| result.response) + Ok(UploadedAttachment { source, thumbnail }) } /// Creates the inner [`MessageType`] for an already-uploaded media file