From 33108d22bc34f6af68f9795076595329376c8fc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Mon, 28 Mar 2022 14:29:56 +0200 Subject: [PATCH] events: Add support for transitional extensible video messages According to MSC3553 --- crates/ruma-common/src/events/file.rs | 9 +- crates/ruma-common/src/events/room/message.rs | 174 +++++++++++++++++- .../src/events/room/message/content_serde.rs | 83 +++++++++ crates/ruma-common/src/events/video.rs | 70 ++++++- crates/ruma-common/tests/events/video.rs | 96 +++++++++- 5 files changed, 422 insertions(+), 10 deletions(-) diff --git a/crates/ruma-common/src/events/file.rs b/crates/ruma-common/src/events/file.rs index 96706be6..ac3b5a25 100644 --- a/crates/ruma-common/src/events/file.rs +++ b/crates/ruma-common/src/events/file.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use super::{ message::MessageContent, room::{ - message::{FileInfo, FileMessageEventContent, Relation}, + message::{FileInfo, FileMessageEventContent, Relation, VideoInfo}, EncryptedFile, ImageInfo, JsonWebKey, MediaSource, }, }; @@ -225,6 +225,13 @@ impl From<&ImageInfo> for FileContentInfo { } } +impl From<&VideoInfo> for FileContentInfo { + fn from(info: &VideoInfo) -> Self { + let VideoInfo { mimetype, size, .. } = info; + Self { mimetype: mimetype.to_owned(), size: size.to_owned(), ..Default::default() } + } +} + /// The encryption info of a file sent to a room with end-to-end encryption enabled. /// /// To create an instance of this type, first create a `EncryptedContentInit` and convert it via diff --git a/crates/ruma-common/src/events/room/message.rs b/crates/ruma-common/src/events/room/message.rs index 56a08fbc..3bda3074 100644 --- a/crates/ruma-common/src/events/room/message.rs +++ b/crates/ruma-common/src/events/room/message.rs @@ -14,6 +14,8 @@ use super::{EncryptedFile, ImageInfo, MediaSource, ThumbnailInfo}; use crate::events::file::{FileContent, FileContentInfo, FileEventContent}; #[cfg(feature = "unstable-msc3552")] use crate::events::image::{ImageContent, ImageEventContent, ThumbnailContent}; +#[cfg(feature = "unstable-msc3553")] +use crate::events::video::{VideoContent, VideoEventContent}; #[cfg(feature = "unstable-msc1767")] use crate::events::{ emote::EmoteEventContent, @@ -247,6 +249,20 @@ impl From for RoomMessageEventContent { } } +#[cfg(feature = "unstable-msc3553")] +impl From for RoomMessageEventContent { + fn from(content: VideoEventContent) -> Self { + let VideoEventContent { message, file, video, thumbnail, caption, relates_to } = content; + + Self { + msgtype: MessageType::Video(VideoMessageEventContent::from_extensible_content( + message, file, video, thumbnail, caption, + )), + relates_to, + } + } +} + /// The content that is specific to each message type variant. #[derive(Clone, Debug, Serialize)] #[serde(untagged)] @@ -1224,6 +1240,10 @@ impl From for TextMessageEventContent { #[derive(Clone, Debug, Deserialize, Serialize)] #[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)] #[serde(tag = "msgtype", rename = "m.video")] +#[cfg_attr( + feature = "unstable-msc3553", + serde(from = "content_serde::VideoMessageEventContentDeHelper") +)] pub struct VideoMessageEventContent { /// A description of the video, e.g. "Gangnam Style", or some kind of content description for /// accessibility, e.g. "video attachment". @@ -1236,19 +1256,128 @@ pub struct VideoMessageEventContent { /// Metadata about the video clip referred to in `url`. #[serde(skip_serializing_if = "Option::is_none")] pub info: Option>, + + /// Extensible-event text representation of the message. + /// + /// If present, this should be preferred over the `body` field. + #[cfg(feature = "unstable-msc3553")] + #[serde(flatten, skip_serializing_if = "Option::is_none")] + pub message: Option, + + /// Extensible-event file content of the message. + /// + /// If present, this should be preferred over the `source` and `info` fields. + #[cfg(feature = "unstable-msc3553")] + #[serde(rename = "org.matrix.msc1767.file", skip_serializing_if = "Option::is_none")] + pub file: Option, + + /// Extensible-event video info of the message. + /// + /// If present, this should be preferred over the `info` field. + #[cfg(feature = "unstable-msc3553")] + #[serde(rename = "org.matrix.msc1767.video", skip_serializing_if = "Option::is_none")] + pub video: Option>, + + /// Extensible-event thumbnails of the message. + /// + /// If present, this should be preferred over the `info` field. + #[cfg(feature = "unstable-msc3553")] + #[serde(rename = "org.matrix.msc1767.thumbnail", skip_serializing_if = "Option::is_none")] + pub thumbnail: Option>, + + /// Extensible-event captions of the message. + #[cfg(feature = "unstable-msc3553")] + #[serde( + rename = "org.matrix.msc1767.caption", + with = "crate::events::message::content_serde::as_vec", + default, + skip_serializing_if = "Option::is_none" + )] + pub caption: Option, } impl VideoMessageEventContent { /// Creates a new non-encrypted `RoomVideoMessageEventContent` with the given body, url and /// optional extra info. pub fn plain(body: String, url: Box, info: Option>) -> Self { - Self { body, source: MediaSource::Plain(url), info } + Self { + #[cfg(feature = "unstable-msc3553")] + message: Some(MessageContent::plain(body.clone())), + #[cfg(feature = "unstable-msc3553")] + file: Some(FileContent::plain( + url.clone(), + info.as_deref().map(|info| Box::new(info.into())), + )), + #[cfg(feature = "unstable-msc3553")] + video: Some(Box::new(info.as_deref().map_or_else(VideoContent::default, Into::into))), + #[cfg(feature = "unstable-msc3553")] + thumbnail: info + .as_deref() + .and_then(|info| { + ThumbnailContent::from_room_message_content( + info.thumbnail_source.as_ref(), + info.thumbnail_info.as_deref(), + ) + }) + .map(|thumbnail| vec![thumbnail]), + #[cfg(feature = "unstable-msc3553")] + caption: None, + body, + source: MediaSource::Plain(url), + info, + } } /// Creates a new encrypted `RoomVideoMessageEventContent` with the given body and encrypted /// file. pub fn encrypted(body: String, file: EncryptedFile) -> Self { - Self { body, source: MediaSource::Encrypted(Box::new(file)), info: None } + Self { + #[cfg(feature = "unstable-msc3553")] + message: Some(MessageContent::plain(body.clone())), + #[cfg(feature = "unstable-msc3553")] + file: Some(FileContent::encrypted(file.url.clone(), (&file).into(), None)), + #[cfg(feature = "unstable-msc3553")] + video: Some(Box::new(VideoContent::default())), + #[cfg(feature = "unstable-msc3553")] + thumbnail: None, + #[cfg(feature = "unstable-msc3553")] + caption: None, + body, + source: MediaSource::Encrypted(Box::new(file)), + info: None, + } + } + + /// Create a new `VideoMessageEventContent` with the given message, file info, video info, + /// thumbnails and captions. + #[cfg(feature = "unstable-msc3553")] + pub fn from_extensible_content( + message: MessageContent, + file: FileContent, + video: Box, + thumbnail: Vec, + caption: Option, + ) -> Self { + let body = if let Some(body) = message.find_plain() { + body.to_owned() + } else { + message[0].body.clone() + }; + let source = (&file).into(); + let info = VideoInfo::from_extensible_content(file.info.as_deref(), &video, &thumbnail) + .map(Box::new); + let thumbnail = if thumbnail.is_empty() { None } else { Some(thumbnail) }; + + Self { + message: Some(message), + file: Some(file), + video: Some(video), + thumbnail, + caption, + body, + source, + info, + } } } @@ -1306,6 +1435,47 @@ impl VideoInfo { pub fn new() -> Self { Self::default() } + + /// Create a `VideoInfo` from the given file info, video info and thumbnail. + #[cfg(feature = "unstable-msc3553")] + pub fn from_extensible_content( + file_info: Option<&FileContentInfo>, + video: &VideoContent, + thumbnail: &[ThumbnailContent], + ) -> Option { + if file_info.is_none() && video.is_empty() && thumbnail.is_empty() { + None + } else { + let (mimetype, size) = file_info + .map(|info| (info.mimetype.to_owned(), info.size.to_owned())) + .unwrap_or_default(); + let VideoContent { duration, height, width } = video.to_owned(); + let (thumbnail_source, thumbnail_info) = thumbnail + .get(0) + .map(|thumbnail| { + let source = (&thumbnail.file).into(); + let info = ThumbnailInfo::from_extensible_content( + thumbnail.file.info.as_deref(), + thumbnail.image.as_deref(), + ) + .map(Box::new); + (Some(source), info) + }) + .unwrap_or_default(); + + Some(Self { + duration, + height, + width, + mimetype, + size, + thumbnail_info, + thumbnail_source, + #[cfg(feature = "unstable-msc2448")] + blurhash: None, + }) + } + } } /// The payload for a key verification request message. diff --git a/crates/ruma-common/src/events/room/message/content_serde.rs b/crates/ruma-common/src/events/room/message/content_serde.rs index 6bdb47b0..8d8a6301 100644 --- a/crates/ruma-common/src/events/room/message/content_serde.rs +++ b/crates/ruma-common/src/events/room/message/content_serde.rs @@ -8,6 +8,8 @@ use super::{FileContent, FileInfo, FileMessageEventContent, MediaSource, Message #[cfg(feature = "unstable-msc3552")] use super::{ImageContent, ImageInfo, ImageMessageEventContent, ThumbnailContent}; use super::{MessageType, Relation, RoomMessageEventContent}; +#[cfg(feature = "unstable-msc3553")] +use super::{VideoContent, VideoInfo, VideoMessageEventContent}; use crate::serde::from_raw_json_value; impl<'de> Deserialize<'de> for RoomMessageEventContent { @@ -187,3 +189,84 @@ impl From for ImageMessageEventContent { Self { body, source, info, message, file, image, thumbnail, caption } } } + +/// Helper struct for deserializing `VideoMessageEventContent` with stable and unstable field names. +/// +/// It's not possible to use the `alias` attribute of serde because of +/// https://github.com/serde-rs/serde/issues/1504. +#[derive(Clone, Debug, Deserialize)] +#[cfg(feature = "unstable-msc3553")] +pub struct VideoMessageEventContentDeHelper { + /// A description of the video. + pub body: String, + + /// The source of the video clip. + #[serde(flatten)] + pub source: MediaSource, + + /// Metadata about the video clip referred to in `source`. + pub info: Option>, + + /// Extensible-event text representation of the message. + #[serde(flatten)] + pub message: Option, + + /// Extensible-event file content of the message, with stable name. + #[serde(rename = "m.file")] + pub file_stable: Option, + + /// Extensible-event file content of the message, with unstable name. + #[serde(rename = "org.matrix.msc1767.file")] + pub file_unstable: Option, + + /// Extensible-event video info of the message, with stable name. + #[serde(rename = "m.video")] + pub video_stable: Option>, + + /// Extensible-event video info of the message, with unstable name. + #[serde(rename = "org.matrix.msc1767.video")] + pub video_unstable: Option>, + + /// Extensible-event thumbnails of the message, with stable name. + #[serde(rename = "m.thumbnail")] + pub thumbnail_stable: Option>, + + /// Extensible-event thumbnails of the message, with unstable name. + #[serde(rename = "org.matrix.msc1767.thumbnail")] + pub thumbnail_unstable: Option>, + + /// Extensible-event captions of the message, with stable name. + #[serde(rename = "m.caption")] + pub caption_stable: Option, + + /// Extensible-event captions of the message, with unstable name. + #[serde(rename = "org.matrix.msc1767.caption")] + pub caption_unstable: Option, +} + +#[cfg(feature = "unstable-msc3553")] +impl From for VideoMessageEventContent { + fn from(helper: VideoMessageEventContentDeHelper) -> Self { + let VideoMessageEventContentDeHelper { + body, + source, + info, + message, + file_stable, + file_unstable, + video_stable, + video_unstable, + thumbnail_stable, + thumbnail_unstable, + caption_stable, + caption_unstable, + } = helper; + + let file = file_stable.or(file_unstable); + let video = video_stable.or(video_unstable); + let thumbnail = thumbnail_stable.or(thumbnail_unstable); + let caption = caption_stable.or(caption_unstable); + + Self { body, source, info, message, file, video, thumbnail, caption } + } +} diff --git a/crates/ruma-common/src/events/video.rs b/crates/ruma-common/src/events/video.rs index c4018403..20dbb67d 100644 --- a/crates/ruma-common/src/events/video.rs +++ b/crates/ruma-common/src/events/video.rs @@ -9,10 +9,25 @@ use ruma_macros::EventContent; use serde::{Deserialize, Serialize}; use super::{ - file::FileContent, image::ThumbnailContent, message::MessageContent, room::message::Relation, + file::FileContent, + image::ThumbnailContent, + message::MessageContent, + room::message::{Relation, VideoInfo, VideoMessageEventContent}, }; /// The payload for an extensible video message. +/// +/// This is the new primary type introduced in [MSC3553] and should not be sent before the end of +/// the transition period. See the documentation of the [`message`] module for more information. +/// +/// `VideoEventContent` can be converted to a [`RoomMessageEventContent`] with a +/// [`MessageType::Video`]. You can convert it back with +/// [`VideoEventContent::from_video_room_message()`]. +/// +/// [MSC3553]: https://github.com/matrix-org/matrix-spec-proposals/pull/3553 +/// [`message`]: super::message +/// [`RoomMessageEventContent`]: super::room::message::RoomMessageEventContent +/// [`MessageType::Video`]: super::room::message::MessageType::Video #[derive(Clone, Debug, Serialize, Deserialize, EventContent)] #[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)] #[ruma_event(type = "m.video", kind = MessageLike)] @@ -71,6 +86,45 @@ impl VideoEventContent { relates_to: None, } } + + /// Create a new `VideoEventContent` from the given `VideoMessageEventContent` and optional + /// relation. + pub fn from_video_room_message( + content: VideoMessageEventContent, + relates_to: Option, + ) -> Self { + let VideoMessageEventContent { + body, + source, + info, + message, + file, + video, + thumbnail, + caption, + } = content; + + let message = message.unwrap_or_else(|| MessageContent::plain(body)); + let file = file.unwrap_or_else(|| { + FileContent::from_room_message_content(source, info.as_deref(), None) + }); + let video = + video.or_else(|| info.as_deref().map(|info| Box::new(info.into()))).unwrap_or_default(); + let thumbnail = thumbnail + .or_else(|| { + info.as_deref() + .and_then(|info| { + ThumbnailContent::from_room_message_content( + info.thumbnail_source.as_ref(), + info.thumbnail_info.as_deref(), + ) + }) + .map(|thumbnail| vec![thumbnail]) + }) + .unwrap_or_default(); + + Self { message, file, video, thumbnail, caption, relates_to } + } } /// Video content. @@ -87,7 +141,7 @@ pub struct VideoContent { /// The duration of the video in milliseconds. #[serde( - with = "ruma_common::serde::duration::opt_ms", + with = "crate::serde::duration::opt_ms", default, skip_serializing_if = "Option::is_none" )] @@ -99,4 +153,16 @@ impl VideoContent { pub fn new() -> Self { Self::default() } + + /// Whether this `VideoContent` is empty. + pub fn is_empty(&self) -> bool { + self.height.is_none() && self.width.is_none() && self.duration.is_none() + } +} + +impl From<&VideoInfo> for VideoContent { + fn from(info: &VideoInfo) -> Self { + let VideoInfo { height, width, duration, .. } = info; + Self { height: height.to_owned(), width: width.to_owned(), duration: duration.to_owned() } + } } diff --git a/crates/ruma-common/tests/events/video.rs b/crates/ruma-common/tests/events/video.rs index 4778b18b..3a571103 100644 --- a/crates/ruma-common/tests/events/video.rs +++ b/crates/ruma-common/tests/events/video.rs @@ -12,8 +12,10 @@ use ruma_common::{ image::{ThumbnailContent, ThumbnailFileContent, ThumbnailFileContentInfo}, message::MessageContent, room::{ - message::{InReplyTo, Relation}, - JsonWebKeyInit, + message::{ + InReplyTo, MessageType, Relation, RoomMessageEventContent, VideoMessageEventContent, + }, + JsonWebKeyInit, MediaSource, }, video::{VideoContent, VideoEventContent}, AnyMessageLikeEvent, MessageLikeEvent, MessageLikeUnsigned, @@ -198,7 +200,7 @@ fn event_serialization() { #[test] fn plain_content_deserialization() { let json_data = json!({ - "org.matrix.msc1767.text": "Video: my_cat.mp4", + "m.text": "Video: my_cat.mp4", "m.file": { "url": "mxc://notareal.hs/abcdef", }, @@ -230,7 +232,7 @@ fn plain_content_deserialization() { #[test] fn encrypted_content_deserialization() { let json_data = json!({ - "org.matrix.msc1767.text": "Upload: my_cat.mp4", + "m.text": "Upload: my_cat.mp4", "m.file": { "url": "mxc://notareal.hs/abcdef", "key": { @@ -273,7 +275,7 @@ fn encrypted_content_deserialization() { fn message_event_deserialization() { let json_data = json!({ "content": { - "org.matrix.msc1767.text": "Upload: my_gnome.webm", + "m.text": "Upload: my_gnome.webm", "m.file": { "url": "mxc://notareal.hs/abcdef", "name": "my_gnome.webm", @@ -329,3 +331,87 @@ fn message_event_deserialization() { && unsigned.is_empty() ); } + +#[test] +fn room_message_serialization() { + let message_event_content = + RoomMessageEventContent::new(MessageType::Video(VideoMessageEventContent::plain( + "Upload: my_video.mp4".to_owned(), + mxc_uri!("mxc://notareal.hs/file").to_owned(), + None, + ))); + + assert_eq!( + to_json_value(&message_event_content).unwrap(), + json!({ + "body": "Upload: my_video.mp4", + "url": "mxc://notareal.hs/file", + "msgtype": "m.video", + "org.matrix.msc1767.text": "Upload: my_video.mp4", + "org.matrix.msc1767.file": { + "url": "mxc://notareal.hs/file", + }, + "org.matrix.msc1767.video": {}, + }) + ); +} + +#[test] +fn room_message_stable_deserialization() { + let json_data = json!({ + "body": "Upload: my_video.mp4", + "url": "mxc://notareal.hs/file", + "msgtype": "m.video", + "m.text": "Upload: my_video.mp4", + "m.file": { + "url": "mxc://notareal.hs/file", + }, + "m.video": {}, + }); + + let event_content = from_json_value::(json_data).unwrap(); + assert_matches!(event_content.msgtype, MessageType::Video(_)); + if let MessageType::Video(content) = event_content.msgtype { + assert_eq!(content.body, "Upload: my_video.mp4"); + assert_matches!(content.source, MediaSource::Plain(_)); + if let MediaSource::Plain(url) = content.source { + assert_eq!(url, "mxc://notareal.hs/file"); + } + let message = content.message.unwrap(); + assert_eq!(message.len(), 1); + assert_eq!(message[0].body, "Upload: my_video.mp4"); + let file = content.file.unwrap(); + assert_eq!(file.url, "mxc://notareal.hs/file"); + assert!(!file.is_encrypted()); + } +} + +#[test] +fn room_message_unstable_deserialization() { + let json_data = json!({ + "body": "Upload: my_video.mp4", + "url": "mxc://notareal.hs/file", + "msgtype": "m.video", + "org.matrix.msc1767.text": "Upload: my_video.mp4", + "org.matrix.msc1767.file": { + "url": "mxc://notareal.hs/file", + }, + "org.matrix.msc1767.video": {}, + }); + + let event_content = from_json_value::(json_data).unwrap(); + assert_matches!(event_content.msgtype, MessageType::Video(_)); + if let MessageType::Video(content) = event_content.msgtype { + assert_eq!(content.body, "Upload: my_video.mp4"); + assert_matches!(content.source, MediaSource::Plain(_)); + if let MediaSource::Plain(url) = content.source { + assert_eq!(url, "mxc://notareal.hs/file"); + } + let message = content.message.unwrap(); + assert_eq!(message.len(), 1); + assert_eq!(message[0].body, "Upload: my_video.mp4"); + let file = content.file.unwrap(); + assert_eq!(file.url, "mxc://notareal.hs/file"); + assert!(!file.is_encrypted()); + } +}