From e94a8db7f4e1a265533321289f4ee6eb72f9d961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Mon, 28 Mar 2022 14:29:14 +0200 Subject: [PATCH] events: Add support for transitional extensible image messages According to MSC3552 --- crates/ruma-common/src/events/file.rs | 9 +- crates/ruma-common/src/events/image.rs | 117 ++++++++++++++- crates/ruma-common/src/events/room.rs | 75 ++++++++++ crates/ruma-common/src/events/room/message.rs | 133 +++++++++++++++++- .../src/events/room/message/content_serde.rs | 83 +++++++++++ crates/ruma-common/tests/events/image.rs | 92 +++++++++++- 6 files changed, 502 insertions(+), 7 deletions(-) diff --git a/crates/ruma-common/src/events/file.rs b/crates/ruma-common/src/events/file.rs index d8467413..96706be6 100644 --- a/crates/ruma-common/src/events/file.rs +++ b/crates/ruma-common/src/events/file.rs @@ -12,7 +12,7 @@ use super::{ message::MessageContent, room::{ message::{FileInfo, FileMessageEventContent, Relation}, - EncryptedFile, JsonWebKey, MediaSource, + EncryptedFile, ImageInfo, JsonWebKey, MediaSource, }, }; use crate::{serde::Base64, MxcUri}; @@ -218,6 +218,13 @@ impl From<&FileInfo> for FileContentInfo { } } +impl From<&ImageInfo> for FileContentInfo { + fn from(info: &ImageInfo) -> Self { + let ImageInfo { mimetype, size, .. } = info; + Self { mimetype: mimetype.to_owned(), size: size.to_owned(), ..Default::default() } + } +} + /// The encryption info of a file sent to a room with end-to-end encryption enabled. /// /// To create an instance of this type, first create a `EncryptedContentInit` and convert it via diff --git a/crates/ruma-common/src/events/image.rs b/crates/ruma-common/src/events/image.rs index 2d01d7d8..7a1c2af8 100644 --- a/crates/ruma-common/src/events/image.rs +++ b/crates/ruma-common/src/events/image.rs @@ -9,11 +9,26 @@ use serde::{Deserialize, Serialize}; use super::{ file::{EncryptedContent, FileContent}, message::MessageContent, - room::message::Relation, + room::{ + message::{ImageMessageEventContent, Relation}, + ImageInfo, MediaSource, ThumbnailInfo, + }, }; use crate::MxcUri; /// The payload for an extensible image message. +/// +/// This is the new primary type introduced in [MSC3552] and should not be sent before the end of +/// the transition period. See the documentation of the [`message`] module for more information. +/// +/// `ImageEventContent` can be converted to a [`RoomMessageEventContent`] with a +/// [`MessageType::Image`]. You can convert it back with +/// [`ImageEventContent::from_image_room_message()`]. +/// +/// [MSC3552]: https://github.com/matrix-org/matrix-spec-proposals/pull/3552 +/// [`message`]: super::message +/// [`RoomMessageEventContent`]: super::room::message::RoomMessageEventContent +/// [`MessageType::Image`]: super::room::message::MessageType::Image #[derive(Clone, Debug, Serialize, Deserialize, EventContent)] #[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)] #[ruma_event(type = "m.image", kind = MessageLike)] @@ -74,6 +89,45 @@ impl ImageEventContent { relates_to: None, } } + + /// Create a new `ImageEventContent` from the given `ImageMessageEventContent` and optional + /// relation. + pub fn from_image_room_message( + content: ImageMessageEventContent, + relates_to: Option, + ) -> Self { + let ImageMessageEventContent { + body, + source, + info, + message, + file, + image, + thumbnail, + caption, + } = content; + + let message = message.unwrap_or_else(|| MessageContent::plain(body)); + let file = file.unwrap_or_else(|| { + FileContent::from_room_message_content(source, info.as_deref(), None) + }); + let image = + image.or_else(|| info.as_deref().map(|info| Box::new(info.into()))).unwrap_or_default(); + let thumbnail = thumbnail + .or_else(|| { + info.as_deref() + .and_then(|info| { + ThumbnailContent::from_room_message_content( + info.thumbnail_source.as_ref(), + info.thumbnail_info.as_deref(), + ) + }) + .map(|thumbnail| vec![thumbnail]) + }) + .unwrap_or_default(); + + Self { message, file, image, thumbnail, caption, relates_to } + } } /// Image content. @@ -99,6 +153,25 @@ impl ImageContent { pub fn with_size(width: UInt, height: UInt) -> Self { Self { height: Some(height), width: Some(width) } } + + /// Whether this `ImageContent` is empty. + pub fn is_empty(&self) -> bool { + self.height.is_none() && self.width.is_none() + } +} + +impl From<&ImageInfo> for ImageContent { + fn from(info: &ImageInfo) -> Self { + let ImageInfo { height, width, .. } = info; + Self { height: height.to_owned(), width: width.to_owned() } + } +} + +impl From<&ThumbnailInfo> for ImageContent { + fn from(info: &ThumbnailInfo) -> Self { + let ThumbnailInfo { height, width, .. } = info; + Self { height: height.to_owned(), width: width.to_owned() } + } } /// Thumbnail content. @@ -119,6 +192,22 @@ impl ThumbnailContent { pub fn new(file: ThumbnailFileContent, image: Option>) -> Self { Self { file, image } } + + /// Create a `ThumbnailContent` with the given thumbnail source and info. + /// + /// Returns `None` if no thumbnail was found. + pub fn from_room_message_content( + thumbnail_source: Option<&MediaSource>, + thumbnail_info: Option<&ThumbnailInfo>, + ) -> Option { + thumbnail_source.map(|thumbnail_source| { + let file = + ThumbnailFileContent::from_room_message_content(thumbnail_source, thumbnail_info); + let image = thumbnail_info.map(|info| Box::new(info.into())); + + Self { file, image } + }) + } } /// Thumbnail file content. @@ -155,6 +244,25 @@ impl ThumbnailFileContent { Self { url, info, encryption_info: Some(Box::new(encryption_info)) } } + /// Create a `ThumbnailContent` with the given thumbnail source and info. + /// + /// Returns `None` if no thumbnail was found. + fn from_room_message_content( + thumbnail_source: &MediaSource, + thumbnail_info: Option<&ThumbnailInfo>, + ) -> Self { + match thumbnail_source { + MediaSource::Plain(url) => { + Self::plain(url.to_owned(), thumbnail_info.map(|info| Box::new(info.into()))) + } + MediaSource::Encrypted(file) => Self::encrypted( + file.url.clone(), + (&**file).into(), + thumbnail_info.map(|info| Box::new(info.into())), + ), + } + } + /// Whether the thumbnail file is encrypted. pub fn is_encrypted(&self) -> bool { self.encryption_info.is_some() @@ -180,3 +288,10 @@ impl ThumbnailFileContentInfo { Self::default() } } + +impl From<&ThumbnailInfo> for ThumbnailFileContentInfo { + fn from(info: &ThumbnailInfo) -> Self { + let ThumbnailInfo { mimetype, size, .. } = info; + Self { mimetype: mimetype.to_owned(), size: size.to_owned() } + } +} diff --git a/crates/ruma-common/src/events/room.rs b/crates/ruma-common/src/events/room.rs index 8dd6c8e2..78a47333 100644 --- a/crates/ruma-common/src/events/room.rs +++ b/crates/ruma-common/src/events/room.rs @@ -9,6 +9,11 @@ use serde::{de, Deserialize, Serialize}; #[cfg(feature = "unstable-msc3551")] use super::file::{EncryptedContent, FileContent}; +#[cfg(feature = "unstable-msc3552")] +use super::{ + file::FileContentInfo, + image::{ImageContent, ThumbnailContent, ThumbnailFileContent, ThumbnailFileContentInfo}, +}; use crate::{ serde::{base64::UrlSafe, Base64}, MxcUri, @@ -84,6 +89,18 @@ impl From<&FileContent> for MediaSource { } } +#[cfg(feature = "unstable-msc3552")] +impl From<&ThumbnailFileContent> for MediaSource { + fn from(content: &ThumbnailFileContent) -> Self { + let ThumbnailFileContent { url, encryption_info, .. } = content; + if let Some(encryption_info) = encryption_info.as_deref() { + Self::Encrypted(Box::new(EncryptedFile::from_extensible_content(url, encryption_info))) + } else { + Self::Plain(url.to_owned()) + } + } +} + /// Metadata about an image. #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)] @@ -130,6 +147,46 @@ impl ImageInfo { pub fn new() -> Self { Self::default() } + + /// Create an `ImageInfo` from the given file info, image info and thumbnail. + #[cfg(feature = "unstable-msc3552")] + pub fn from_extensible_content( + file_info: Option<&FileContentInfo>, + image: &ImageContent, + thumbnail: &[ThumbnailContent], + ) -> Option { + if file_info.is_none() && image.is_empty() && thumbnail.is_empty() { + None + } else { + let (mimetype, size) = file_info + .map(|info| (info.mimetype.to_owned(), info.size.to_owned())) + .unwrap_or_default(); + let ImageContent { height, width } = image.to_owned(); + let (thumbnail_source, thumbnail_info) = thumbnail + .get(0) + .map(|thumbnail| { + let source = (&thumbnail.file).into(); + let info = ThumbnailInfo::from_extensible_content( + thumbnail.file.info.as_deref(), + thumbnail.image.as_deref(), + ) + .map(Box::new); + (Some(source), info) + }) + .unwrap_or_default(); + + Some(Self { + height, + width, + mimetype, + size, + thumbnail_source, + thumbnail_info, + #[cfg(feature = "unstable-msc2448")] + blurhash: None, + }) + } + } } /// Metadata about a thumbnail. @@ -158,6 +215,24 @@ impl ThumbnailInfo { pub fn new() -> Self { Self::default() } + + /// Create a `ThumbnailInfo` with the given file info and image info. + /// + /// Returns `None` if `file_info` and `image` are `None`. + #[cfg(feature = "unstable-msc3552")] + pub fn from_extensible_content( + file_info: Option<&ThumbnailFileContentInfo>, + image: Option<&ImageContent>, + ) -> Option { + if file_info.is_none() && image.is_none() { + None + } else { + let ThumbnailFileContentInfo { mimetype, size } = + file_info.map(ToOwned::to_owned).unwrap_or_default(); + let ImageContent { height, width } = image.map(ToOwned::to_owned).unwrap_or_default(); + Some(Self { height, width, mimetype, size }) + } + } } /// A file sent to a room with end-to-end encryption enabled. diff --git a/crates/ruma-common/src/events/room/message.rs b/crates/ruma-common/src/events/room/message.rs index 5773637f..56a08fbc 100644 --- a/crates/ruma-common/src/events/room/message.rs +++ b/crates/ruma-common/src/events/room/message.rs @@ -12,6 +12,8 @@ use serde_json::Value as JsonValue; use super::{EncryptedFile, ImageInfo, MediaSource, ThumbnailInfo}; #[cfg(feature = "unstable-msc3551")] use crate::events::file::{FileContent, FileContentInfo, FileEventContent}; +#[cfg(feature = "unstable-msc3552")] +use crate::events::image::{ImageContent, ImageEventContent, ThumbnailContent}; #[cfg(feature = "unstable-msc1767")] use crate::events::{ emote::EmoteEventContent, @@ -213,6 +215,20 @@ impl From for RoomMessageEventContent { } } +#[cfg(feature = "unstable-msc3552")] +impl From for RoomMessageEventContent { + fn from(content: ImageEventContent) -> Self { + let ImageEventContent { message, file, image, thumbnail, caption, relates_to } = content; + + Self { + msgtype: MessageType::Image(ImageMessageEventContent::from_extensible_content( + message, file, image, thumbnail, caption, + )), + relates_to, + } + } +} + #[cfg(feature = "unstable-msc1767")] impl From for RoomMessageEventContent { fn from(content: MessageEventContent) -> Self { @@ -744,6 +760,10 @@ impl From<&FileContentInfo> for FileInfo { #[derive(Clone, Debug, Deserialize, Serialize)] #[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)] #[serde(tag = "msgtype", rename = "m.image")] +#[cfg_attr( + feature = "unstable-msc3552", + serde(from = "content_serde::ImageMessageEventContentDeHelper") +)] pub struct ImageMessageEventContent { /// A textual representation of the image. /// @@ -758,19 +778,128 @@ pub struct ImageMessageEventContent { /// Metadata about the image referred to in `url`. #[serde(skip_serializing_if = "Option::is_none")] pub info: Option>, + + /// Extensible-event text representation of the message. + /// + /// If present, this should be preferred over the `body` field. + #[cfg(feature = "unstable-msc3552")] + #[serde(flatten, skip_serializing_if = "Option::is_none")] + pub message: Option, + + /// Extensible-event file content of the message. + /// + /// If present, this should be preferred over the `source` and `info` fields. + #[cfg(feature = "unstable-msc3552")] + #[serde(rename = "org.matrix.msc1767.file", skip_serializing_if = "Option::is_none")] + pub file: Option, + + /// Extensible-event image info of the message. + /// + /// If present, this should be preferred over the `info` field. + #[cfg(feature = "unstable-msc3552")] + #[serde(rename = "org.matrix.msc1767.image", skip_serializing_if = "Option::is_none")] + pub image: Option>, + + /// Extensible-event thumbnails of the message. + /// + /// If present, this should be preferred over the `info` field. + #[cfg(feature = "unstable-msc3552")] + #[serde(rename = "org.matrix.msc1767.thumbnail", skip_serializing_if = "Option::is_none")] + pub thumbnail: Option>, + + /// Extensible-event captions of the message. + #[cfg(feature = "unstable-msc3552")] + #[serde( + rename = "org.matrix.msc1767.caption", + with = "crate::events::message::content_serde::as_vec", + default, + skip_serializing_if = "Option::is_none" + )] + pub caption: Option, } impl ImageMessageEventContent { /// Creates a new non-encrypted `RoomImageMessageEventContent` with the given body, url and /// optional extra info. pub fn plain(body: String, url: Box, info: Option>) -> Self { - Self { body, source: MediaSource::Plain(url), info } + Self { + #[cfg(feature = "unstable-msc3552")] + message: Some(MessageContent::plain(body.clone())), + #[cfg(feature = "unstable-msc3552")] + file: Some(FileContent::plain( + url.clone(), + info.as_deref().map(|info| Box::new(info.into())), + )), + #[cfg(feature = "unstable-msc3552")] + image: Some(Box::new(info.as_deref().map_or_else(ImageContent::default, Into::into))), + #[cfg(feature = "unstable-msc3552")] + thumbnail: info + .as_deref() + .and_then(|info| { + ThumbnailContent::from_room_message_content( + info.thumbnail_source.as_ref(), + info.thumbnail_info.as_deref(), + ) + }) + .map(|thumbnail| vec![thumbnail]), + #[cfg(feature = "unstable-msc3552")] + caption: None, + body, + source: MediaSource::Plain(url), + info, + } } /// Creates a new encrypted `RoomImageMessageEventContent` with the given body and encrypted /// file. pub fn encrypted(body: String, file: EncryptedFile) -> Self { - Self { body, source: MediaSource::Encrypted(Box::new(file)), info: None } + Self { + #[cfg(feature = "unstable-msc3552")] + message: Some(MessageContent::plain(body.clone())), + #[cfg(feature = "unstable-msc3552")] + file: Some(FileContent::encrypted(file.url.clone(), (&file).into(), None)), + #[cfg(feature = "unstable-msc3552")] + image: Some(Box::new(ImageContent::default())), + #[cfg(feature = "unstable-msc3552")] + thumbnail: None, + #[cfg(feature = "unstable-msc3552")] + caption: None, + body, + source: MediaSource::Encrypted(Box::new(file)), + info: None, + } + } + + /// Create a new `ImageMessageEventContent` with the given message, file info, image info, + /// thumbnails and captions. + #[cfg(feature = "unstable-msc3552")] + pub fn from_extensible_content( + message: MessageContent, + file: FileContent, + image: Box, + thumbnail: Vec, + caption: Option, + ) -> Self { + let body = if let Some(body) = message.find_plain() { + body.to_owned() + } else { + message[0].body.clone() + }; + let source = (&file).into(); + let info = ImageInfo::from_extensible_content(file.info.as_deref(), &image, &thumbnail) + .map(Box::new); + let thumbnail = if thumbnail.is_empty() { None } else { Some(thumbnail) }; + + Self { + message: Some(message), + file: Some(file), + image: Some(image), + thumbnail, + caption, + body, + source, + info, + } } } diff --git a/crates/ruma-common/src/events/room/message/content_serde.rs b/crates/ruma-common/src/events/room/message/content_serde.rs index 7fe6183a..6bdb47b0 100644 --- a/crates/ruma-common/src/events/room/message/content_serde.rs +++ b/crates/ruma-common/src/events/room/message/content_serde.rs @@ -5,6 +5,8 @@ use serde_json::value::RawValue as RawJsonValue; #[cfg(feature = "unstable-msc3551")] use super::{FileContent, FileInfo, FileMessageEventContent, MediaSource, MessageContent}; +#[cfg(feature = "unstable-msc3552")] +use super::{ImageContent, ImageInfo, ImageMessageEventContent, ThumbnailContent}; use super::{MessageType, Relation, RoomMessageEventContent}; use crate::serde::from_raw_json_value; @@ -104,3 +106,84 @@ impl From for FileMessageEventContent { Self { body, filename, source, info, message, file } } } + +/// Helper struct for deserializing `ImageMessageEventContent` with stable and unstable field names. +/// +/// It's not possible to use the `alias` attribute of serde because of +/// https://github.com/serde-rs/serde/issues/1504. +#[derive(Clone, Debug, Deserialize)] +#[cfg(feature = "unstable-msc3552")] +pub struct ImageMessageEventContentDeHelper { + /// A textual representation of the image. + pub body: String, + + /// The source of the image. + #[serde(flatten)] + pub source: MediaSource, + + /// Metadata about the image referred to in `source`. + pub info: Option>, + + /// Extensible-event text representation of the message. + #[serde(flatten)] + pub message: Option, + + /// Extensible-event file content of the message, with unstable name. + #[serde(rename = "m.file")] + pub file_stable: Option, + + /// Extensible-event file content of the message, with unstable name. + #[serde(rename = "org.matrix.msc1767.file")] + pub file_unstable: Option, + + /// Extensible-event image info of the message, with stable name. + #[serde(rename = "m.image")] + pub image_stable: Option>, + + /// Extensible-event image info of the message, with unstable name. + #[serde(rename = "org.matrix.msc1767.image")] + pub image_unstable: Option>, + + /// Extensible-event thumbnails of the message, with stable name. + #[serde(rename = "m.thumbnail")] + pub thumbnail_stable: Option>, + + /// Extensible-event thumbnails of the message, with unstable name. + #[serde(rename = "org.matrix.msc1767.thumbnail")] + pub thumbnail_unstable: Option>, + + /// Extensible-event captions of the message, with stable name. + #[serde(rename = "m.caption")] + pub caption_stable: Option, + + /// Extensible-event captions of the message, with unstable name. + #[serde(rename = "org.matrix.msc1767.caption")] + pub caption_unstable: Option, +} + +#[cfg(feature = "unstable-msc3552")] +impl From for ImageMessageEventContent { + fn from(helper: ImageMessageEventContentDeHelper) -> Self { + let ImageMessageEventContentDeHelper { + body, + source, + info, + message, + file_stable, + file_unstable, + image_stable, + image_unstable, + thumbnail_stable, + thumbnail_unstable, + caption_stable, + caption_unstable, + } = helper; + + let file = file_stable.or(file_unstable); + let image = image_stable.or(image_unstable); + let thumbnail = thumbnail_stable.or(thumbnail_unstable); + let caption = caption_stable.or(caption_unstable); + + Self { body, source, info, message, file, image, thumbnail, caption } + } +} diff --git a/crates/ruma-common/tests/events/image.rs b/crates/ruma-common/tests/events/image.rs index 110d546d..00428d6e 100644 --- a/crates/ruma-common/tests/events/image.rs +++ b/crates/ruma-common/tests/events/image.rs @@ -13,8 +13,10 @@ use ruma_common::{ }, message::MessageContent, room::{ - message::{InReplyTo, Relation}, - JsonWebKeyInit, + message::{ + ImageMessageEventContent, InReplyTo, MessageType, Relation, RoomMessageEventContent, + }, + JsonWebKeyInit, MediaSource, }, AnyMessageLikeEvent, MessageLikeEvent, MessageLikeUnsigned, }, @@ -190,7 +192,7 @@ fn image_event_serialization() { #[test] fn plain_content_deserialization() { let json_data = json!({ - "org.matrix.msc1767.text": "Upload: my_cat.png", + "m.text": "Upload: my_cat.png", "m.file": { "url": "mxc://notareal.hs/abcdef", }, @@ -318,3 +320,87 @@ fn message_event_deserialization() { && unsigned.is_empty() ); } + +#[test] +fn room_message_serialization() { + let message_event_content = + RoomMessageEventContent::new(MessageType::Image(ImageMessageEventContent::plain( + "Upload: my_image.jpg".to_owned(), + mxc_uri!("mxc://notareal.hs/file").to_owned(), + None, + ))); + + assert_eq!( + to_json_value(&message_event_content).unwrap(), + json!({ + "body": "Upload: my_image.jpg", + "url": "mxc://notareal.hs/file", + "msgtype": "m.image", + "org.matrix.msc1767.text": "Upload: my_image.jpg", + "org.matrix.msc1767.file": { + "url": "mxc://notareal.hs/file", + }, + "org.matrix.msc1767.image": {}, + }) + ); +} + +#[test] +fn room_message_stable_deserialization() { + let json_data = json!({ + "body": "Upload: my_image.jpg", + "url": "mxc://notareal.hs/file", + "msgtype": "m.image", + "m.text": "Upload: my_image.jpg", + "m.file": { + "url": "mxc://notareal.hs/file", + }, + "m.image": {}, + }); + + let event_content = from_json_value::(json_data).unwrap(); + assert_matches!(event_content.msgtype, MessageType::Image(_)); + if let MessageType::Image(content) = event_content.msgtype { + assert_eq!(content.body, "Upload: my_image.jpg"); + assert_matches!(content.source, MediaSource::Plain(_)); + if let MediaSource::Plain(url) = content.source { + assert_eq!(url, "mxc://notareal.hs/file"); + } + let message = content.message.unwrap(); + assert_eq!(message.len(), 1); + assert_eq!(message[0].body, "Upload: my_image.jpg"); + let file = content.file.unwrap(); + assert_eq!(file.url, "mxc://notareal.hs/file"); + assert!(!file.is_encrypted()); + } +} + +#[test] +fn room_message_unstable_deserialization() { + let json_data = json!({ + "body": "Upload: my_image.jpg", + "url": "mxc://notareal.hs/file", + "msgtype": "m.image", + "org.matrix.msc1767.text": "Upload: my_image.jpg", + "org.matrix.msc1767.file": { + "url": "mxc://notareal.hs/file", + }, + "org.matrix.msc1767.image": {}, + }); + + let event_content = from_json_value::(json_data).unwrap(); + assert_matches!(event_content.msgtype, MessageType::Image(_)); + if let MessageType::Image(content) = event_content.msgtype { + assert_eq!(content.body, "Upload: my_image.jpg"); + assert_matches!(content.source, MediaSource::Plain(_)); + if let MediaSource::Plain(url) = content.source { + assert_eq!(url, "mxc://notareal.hs/file"); + } + let message = content.message.unwrap(); + assert_eq!(message.len(), 1); + assert_eq!(message[0].body, "Upload: my_image.jpg"); + let file = content.file.unwrap(); + assert_eq!(file.url, "mxc://notareal.hs/file"); + assert!(!file.is_encrypted()); + } +}