common: Add support for extensible audio events

crates/ruma-common/Cargo.toml

@@ -33,6 +33,7 @@ unstable-msc2448 = []
 unstable-msc2675 = []
 unstable-msc2676 = []
 unstable-msc2677 = []
+unstable-msc3246 = ["unstable-msc3551", "thiserror"]
 unstable-msc3551 = ["unstable-msc1767"]
 unstable-msc3552 = ["unstable-msc1767", "unstable-msc3551"]
 unstable-msc3553 = ["unstable-msc3552"]

crates/ruma-common/src/events.rs

@@ -144,6 +144,8 @@ pub mod macros {
     pub use ruma_macros::{Event, EventContent};
 }
 
+#[cfg(feature = "unstable-msc3246")]
+pub mod audio;
 pub mod call;
 pub mod direct;
 pub mod dummy;

crates/ruma-common/src/events/audio.rs

@@ -0,0 +1,167 @@
+//! Types for extensible audio message events ([MSC3246]).
+//!
+//! [MSC3246]: https://github.com/matrix-org/matrix-spec-proposals/pull/3246
+
+use std::{convert::TryFrom, time::Duration};
+
+use js_int::UInt;
+use ruma_macros::EventContent;
+use serde::{Deserialize, Serialize};
+
+mod amplitude_serde;
+mod waveform_serde;
+
+use waveform_serde::WaveformSerDeHelper;
+
+use super::{file::FileContent, message::MessageContent, room::message::Relation};
+
+/// The payload for an extensible audio message.
+#[derive(Clone, Debug, Serialize, Deserialize, EventContent)]
+#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
+#[ruma_event(type = "m.audio", kind = MessageLike)]
+pub struct AudioEventContent {
+    /// The text representation of the message.
+    #[serde(flatten)]
+    pub message: MessageContent,
+
+    /// The file content of the message.
+    #[serde(rename = "org.matrix.msc1767.file")]
+    pub file: FileContent,
+
+    /// The audio content of the message.
+    #[serde(rename = "org.matrix.msc1767.audio")]
+    pub audio: AudioContent,
+
+    /// Information about related messages.
+    #[serde(flatten, skip_serializing_if = "Option::is_none")]
+    pub relates_to: Option<Relation>,
+}
+
+impl AudioEventContent {
+    /// Creates a new `AudioEventContent` with the given plain text message and file.
+    pub fn plain(message: impl Into<String>, file: FileContent) -> Self {
+        Self {
+            message: MessageContent::plain(message),
+            file,
+            audio: Default::default(),
+            relates_to: None,
+        }
+    }
+
+    /// Creates a new `AudioEventContent` with the given message and file.
+    pub fn with_message(message: MessageContent, file: FileContent) -> Self {
+        Self { message, file, audio: Default::default(), relates_to: None }
+    }
+}
+
+/// Audio content.
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
+pub struct AudioContent {
+    /// The duration of the video in milliseconds.
+    #[serde(
+        with = "ruma_common::serde::duration::opt_ms",
+        default,
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub duration: Option<Duration>,
+
+    /// The waveform representation of the audio content.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub waveform: Option<Waveform>,
+}
+
+impl AudioContent {
+    /// Creates a new empty `AudioContent`.
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+
+/// The waveform representation of audio content.
+///
+/// Must include between 30 and 120 `Amplitude`s.
+///
+/// To build this, use the `TryFrom` implementations.
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(try_from = "WaveformSerDeHelper")]
+pub struct Waveform(Vec<Amplitude>);
+
+impl Waveform {
+    /// The smallest number of values contained in a `Waveform`.
+    pub const MIN_LENGTH: usize = 30;
+
+    /// The largest number of values contained in a `Waveform`.
+    pub const MAX_LENGTH: usize = 120;
+
+    /// The amplitudes of this `Waveform`.
+    pub fn amplitudes(&self) -> &[Amplitude] {
+        &self.0
+    }
+}
+
+/// An error encountered when trying to convert to a `Waveform`.
+#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq, thiserror::Error)]
+#[non_exhaustive]
+pub enum WaveformError {
+    /// There are more than [`Waveform::MAX`] values.
+    #[error("too many values")]
+    TooManyValues,
+    /// There are less that [`Waveform::MIN`] values.
+    #[error("not enough values")]
+    NotEnoughValues,
+}
+
+impl TryFrom<Vec<Amplitude>> for Waveform {
+    type Error = WaveformError;
+
+    fn try_from(value: Vec<Amplitude>) -> Result<Self, Self::Error> {
+        if value.len() < Self::MIN_LENGTH {
+            Err(WaveformError::NotEnoughValues)
+        } else if value.len() > Self::MAX_LENGTH {
+            Err(WaveformError::TooManyValues)
+        } else {
+            Ok(Self(value))
+        }
+    }
+}
+
+impl TryFrom<&[Amplitude]> for Waveform {
+    type Error = WaveformError;
+
+    fn try_from(value: &[Amplitude]) -> Result<Self, Self::Error> {
+        Self::try_from(value.to_owned())
+    }
+}
+
+/// The amplitude of a waveform sample.
+///
+/// Must be an integer between 0 and 1024.
+#[derive(Clone, Copy, Debug, Default, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize)]
+pub struct Amplitude(UInt);
+
+impl Amplitude {
+    /// The smallest value that can be represented by this type, 0.
+    pub const MIN: u16 = 0;
+
+    /// The largest value that can be represented by this type, 1024.
+    pub const MAX: u16 = 1024;
+
+    /// Creates a new `Amplitude` with the given value.
+    ///
+    /// It will saturate if it is bigger than [`Amplitude::MAX`].
+    pub fn new(value: u16) -> Self {
+        Self(value.min(Self::MAX).into())
+    }
+
+    /// The value of this `Amplitude`.
+    pub fn value(&self) -> UInt {
+        self.0
+    }
+}
+
+impl From<u16> for Amplitude {
+    fn from(value: u16) -> Self {
+        Self::new(value)
+    }
+}

crates/ruma-common/src/events/audio/amplitude_serde.rs

@@ -0,0 +1,16 @@
+//! `Serialize` and `Deserialize` implementations for extensible events (MSC1767).
+
+use js_int::UInt;
+use serde::Deserialize;
+
+use super::Amplitude;
+
+impl<'de> Deserialize<'de> for Amplitude {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let uint = UInt::deserialize(deserializer)?;
+        Ok(Self(uint.min(Self::MAX.into())))
+    }
+}

crates/ruma-common/src/events/audio/waveform_serde.rs

@@ -0,0 +1,18 @@
+//! `Serialize` and `Deserialize` implementations for extensible events (MSC1767).
+
+use std::convert::TryFrom;
+
+use serde::Deserialize;
+
+use super::{Amplitude, Waveform, WaveformError};
+
+#[derive(Debug, Default, Deserialize)]
+pub(crate) struct WaveformSerDeHelper(Vec<Amplitude>);
+
+impl TryFrom<WaveformSerDeHelper> for Waveform {
+    type Error = WaveformError;
+
+    fn try_from(helper: WaveformSerDeHelper) -> Result<Self, Self::Error> {
+        Waveform::try_from(helper.0)
+    }
+}

crates/ruma-common/src/events/enums.rs

@@ -33,6 +33,8 @@ event_enum! {
 
     /// Any message-like event.
     enum MessageLike {
+        #[cfg(feature = "unstable-msc3246")]
+        "m.audio",
         "m.call.answer",
         "m.call.invite",
         "m.call.hangup",
@@ -370,6 +372,8 @@ impl AnyMessageLikeEventContent {
             Self::Notice(ev) => ev.relates_to.clone().map(Into::into),
             #[cfg(feature = "unstable-msc1767")]
             Self::Emote(ev) => ev.relates_to.clone().map(Into::into),
+            #[cfg(feature = "unstable-msc3246")]
+            Self::Audio(ev) => ev.relates_to.clone().map(Into::into),
             #[cfg(feature = "unstable-msc3551")]
             Self::File(ev) => ev.relates_to.clone().map(Into::into),
             #[cfg(feature = "unstable-msc3552")]

crates/ruma-common/tests/events/audio.rs

@@ -0,0 +1,381 @@
+#![cfg(feature = "unstable-msc3246")]
+
+use std::time::Duration;
+
+use assign::assign;
+use js_int::uint;
+use matches::assert_matches;
+use ruma_common::{
+    event_id,
+    events::{
+        audio::{Amplitude, AudioContent, AudioEventContent, Waveform, WaveformError},
+        file::{EncryptedContentInit, FileContent, FileContentInfo},
+        message::MessageContent,
+        room::{
+            message::{InReplyTo, Relation},
+            JsonWebKeyInit,
+        },
+        AnyMessageLikeEvent, MessageLikeEvent, Unsigned,
+    },
+    mxc_uri, room_id,
+    serde::Base64,
+    user_id, MilliSecondsSinceUnixEpoch,
+};
+use serde_json::{from_value as from_json_value, json, to_value as to_json_value};
+
+#[test]
+fn waveform_deserialization_pass() {
+    let json_data = json!([
+        13, 34, 987, 937, 345, 648, 1, 366, 235, 125, 904, 783, 734, 13, 34, 987, 937, 345, 648, 1,
+        366, 235, 125, 904, 783, 734, 13, 34, 987, 937, 345, 648, 1, 366, 235, 125, 904, 783, 734,
+        13, 34, 987, 937, 345, 648, 1, 366, 235, 125, 904, 783, 734,
+    ]);
+
+    assert_matches!(
+        from_json_value::<Waveform>(json_data),
+        Ok(waveform) if waveform.amplitudes().len() == 52
+    );
+}
+
+#[test]
+fn waveform_deserialization_not_enough() {
+    let json_data = json!([]);
+
+    assert_matches!(
+        from_json_value::<Waveform>(json_data),
+        Err(err)
+            if err.is_data()
+            && format!("{}", err) == format!("{}", WaveformError::NotEnoughValues)
+    );
+}
+
+#[test]
+fn waveform_deserialization_clamp_amplitude() {
+    let json_data = json!([
+        2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000,
+        2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000,
+    ]);
+
+    assert_matches!(
+        from_json_value::<Waveform>(json_data).unwrap(),
+        waveform if waveform.amplitudes().iter().all(|amp| amp.value() == Amplitude::MAX.into())
+    );
+}
+
+#[test]
+fn plain_content_serialization() {
+    let event_content = AudioEventContent::plain(
+        "Upload: my_sound.ogg",
+        FileContent::plain(mxc_uri!("mxc://notareal.hs/abcdef").to_owned(), None),
+    );
+
+    assert_eq!(
+        to_json_value(&event_content).unwrap(),
+        json!({
+            "org.matrix.msc1767.text": "Upload: my_sound.ogg",
+            "org.matrix.msc1767.file": {
+                "url": "mxc://notareal.hs/abcdef",
+            },
+            "org.matrix.msc1767.audio": {}
+        })
+    );
+}
+
+#[test]
+fn encrypted_content_serialization() {
+    let event_content = AudioEventContent::plain(
+        "Upload: my_sound.ogg",
+        FileContent::encrypted(
+            mxc_uri!("mxc://notareal.hs/abcdef").to_owned(),
+            EncryptedContentInit {
+                key: JsonWebKeyInit {
+                    kty: "oct".to_owned(),
+                    key_ops: vec!["encrypt".to_owned(), "decrypt".to_owned()],
+                    alg: "A256CTR".to_owned(),
+                    k: Base64::parse("TLlG_OpX807zzQuuwv4QZGJ21_u7weemFGYJFszMn9A").unwrap(),
+                    ext: true,
+                }
+                .into(),
+                iv: Base64::parse("S22dq3NAX8wAAAAAAAAAAA").unwrap(),
+                hashes: [(
+                    "sha256".to_owned(),
+                    Base64::parse("aWOHudBnDkJ9IwaR1Nd8XKoI7DOrqDTwt6xDPfVGN6Q").unwrap(),
+                )]
+                .into(),
+                v: "v2".to_owned(),
+            }
+            .into(),
+            None,
+        ),
+    );
+
+    assert_eq!(
+        to_json_value(&event_content).unwrap(),
+        json!({
+            "org.matrix.msc1767.text": "Upload: my_sound.ogg",
+            "org.matrix.msc1767.file": {
+                "url": "mxc://notareal.hs/abcdef",
+                "key": {
+                    "kty": "oct",
+                    "key_ops": ["encrypt", "decrypt"],
+                    "alg": "A256CTR",
+                    "k": "TLlG_OpX807zzQuuwv4QZGJ21_u7weemFGYJFszMn9A",
+                    "ext": true
+                },
+                "iv": "S22dq3NAX8wAAAAAAAAAAA",
+                "hashes": {
+                    "sha256": "aWOHudBnDkJ9IwaR1Nd8XKoI7DOrqDTwt6xDPfVGN6Q"
+                },
+                "v": "v2"
+            },
+            "org.matrix.msc1767.audio": {}
+        })
+    );
+}
+
+#[test]
+fn event_serialization() {
+    let event = MessageLikeEvent {
+        content: assign!(
+            AudioEventContent::with_message(
+                MessageContent::html(
+                    "Upload: my_mix.mp3",
+                    "Upload: <strong>my_mix.mp3</strong>",
+                ),
+                FileContent::plain(
+                    mxc_uri!("mxc://notareal.hs/abcdef").to_owned(),
+                    Some(Box::new(assign!(
+                        FileContentInfo::new(),
+                        {
+                            name: Some("my_mix.mp3".to_owned()),
+                            mimetype: Some("audio/mp3".to_owned()),
+                            size: Some(uint!(897_774)),
+                        }
+                    ))),
+                )
+            ),
+            {
+                audio: assign!(
+                    AudioContent::new(),
+                    {
+                        duration: Some(Duration::from_secs(123))
+                    }
+                ),
+                relates_to: Some(Relation::Reply {
+                    in_reply_to: InReplyTo::new(event_id!("$replyevent:example.com").to_owned()),
+                }),
+            }
+        ),
+        event_id: event_id!("$event:notareal.hs").to_owned(),
+        sender: user_id!("@user:notareal.hs").to_owned(),
+        origin_server_ts: MilliSecondsSinceUnixEpoch(uint!(134_829_848)),
+        room_id: room_id!("!roomid:notareal.hs").to_owned(),
+        unsigned: Unsigned::default(),
+    };
+
+    assert_eq!(
+        to_json_value(&event).unwrap(),
+        json!({
+            "content": {
+                "org.matrix.msc1767.message": [
+                    { "body": "Upload: <strong>my_mix.mp3</strong>", "mimetype": "text/html"},
+                    { "body": "Upload: my_mix.mp3", "mimetype": "text/plain"},
+                ],
+                "org.matrix.msc1767.file": {
+                    "url": "mxc://notareal.hs/abcdef",
+                    "name": "my_mix.mp3",
+                    "mimetype": "audio/mp3",
+                    "size": 897_774,
+                },
+                "org.matrix.msc1767.audio": {
+                    "duration": 123_000,
+                },
+                "m.relates_to": {
+                    "m.in_reply_to": {
+                        "event_id": "$replyevent:example.com"
+                    }
+                }
+            },
+            "event_id": "$event:notareal.hs",
+            "origin_server_ts": 134_829_848,
+            "room_id": "!roomid:notareal.hs",
+            "sender": "@user:notareal.hs",
+            "type": "m.audio",
+        })
+    );
+}
+
+#[test]
+fn plain_content_deserialization() {
+    let json_data = json!({
+        "org.matrix.msc1767.text": "Upload: my_new_song.webm",
+        "org.matrix.msc1767.file": {
+            "url": "mxc://notareal.hs/abcdef",
+        },
+        "org.matrix.msc1767.audio": {
+            "waveform": [
+                13,
+                34,
+                987,
+                937,
+                345,
+                648,
+                1,
+                366,
+                235,
+                125,
+                904,
+                783,
+                734,
+                13,
+                34,
+                987,
+                937,
+                345,
+                648,
+                1,
+                366,
+                235,
+                125,
+                904,
+                783,
+                734,
+                13,
+                34,
+                987,
+                937,
+                345,
+                648,
+                1,
+                366,
+                235,
+                125,
+                904,
+                783,
+                734,
+                13,
+                34,
+                987,
+                937,
+                345,
+                648,
+                1,
+                366,
+                235,
+                125,
+                904,
+                783,
+                734,
+            ],
+        }
+    });
+
+    assert_matches!(
+        from_json_value::<AudioEventContent>(json_data)
+            .unwrap(),
+        AudioEventContent {
+            message,
+            file,
+            audio: AudioContent { duration: None, waveform: Some(waveform), .. },
+            ..
+        }
+        if message.find_plain() == Some("Upload: my_new_song.webm")
+            && message.find_html().is_none()
+            && file.url == "mxc://notareal.hs/abcdef"
+            && waveform.amplitudes().len() == 52
+    );
+}
+
+#[test]
+fn encrypted_content_deserialization() {
+    let json_data = json!({
+        "org.matrix.msc1767.text": "Upload: my_file.txt",
+        "org.matrix.msc1767.file": {
+            "url": "mxc://notareal.hs/abcdef",
+            "key": {
+                "kty": "oct",
+                "key_ops": ["encrypt", "decrypt"],
+                "alg": "A256CTR",
+                "k": "TLlG_OpX807zzQuuwv4QZGJ21_u7weemFGYJFszMn9A",
+                "ext": true
+            },
+            "iv": "S22dq3NAX8wAAAAAAAAAAA",
+            "hashes": {
+                "sha256": "aWOHudBnDkJ9IwaR1Nd8XKoI7DOrqDTwt6xDPfVGN6Q"
+            },
+            "v": "v2"
+        },
+        "org.matrix.msc1767.audio": {},
+    });
+
+    assert_matches!(
+        from_json_value::<AudioEventContent>(json_data)
+            .unwrap(),
+        AudioEventContent {
+            message,
+            file,
+            audio: AudioContent { duration: None, waveform: None, .. },
+            ..
+        }
+        if message.find_plain() == Some("Upload: my_file.txt")
+            && message.find_html().is_none()
+            && file.url == "mxc://notareal.hs/abcdef"
+            && file.encryption_info.is_some()
+    );
+}
+
+#[test]
+fn message_event_deserialization() {
+    let json_data = json!({
+        "content": {
+            "org.matrix.msc1767.text": "Upload: airplane_sound.opus",
+            "org.matrix.msc1767.file": {
+                "url": "mxc://notareal.hs/abcdef",
+                "name": "airplane_sound.opus",
+                "mimetype": "audio/opus",
+                "size": 123_774,
+            },
+            "org.matrix.msc1767.audio": {
+                "duration": 5_300,
+            }
+        },
+        "event_id": "$event:notareal.hs",
+        "origin_server_ts": 134_829_848,
+        "room_id": "!roomid:notareal.hs",
+        "sender": "@user:notareal.hs",
+        "type": "m.audio",
+    });
+
+    assert_matches!(
+        from_json_value::<AnyMessageLikeEvent>(json_data).unwrap(),
+        AnyMessageLikeEvent::Audio(MessageLikeEvent {
+            content: AudioEventContent {
+                message,
+                file: FileContent {
+                    url,
+                    info: Some(info),
+                    ..
+                },
+                audio,
+                ..
+            },
+            event_id,
+            origin_server_ts,
+            room_id,
+            sender,
+            unsigned
+        }) if event_id == event_id!("$event:notareal.hs")
+            && message.find_plain() == Some("Upload: airplane_sound.opus")
+            && message.find_html().is_none()
+            && url == "mxc://notareal.hs/abcdef"
+            && info.name.as_deref() == Some("airplane_sound.opus")
+            && info.mimetype.as_deref() == Some("audio/opus")
+            && info.size == Some(uint!(123_774))
+            && audio.duration == Some(Duration::from_millis(5_300))
+            && audio.waveform.is_none()
+            && origin_server_ts == MilliSecondsSinceUnixEpoch(uint!(134_829_848))
+            && room_id == room_id!("!roomid:notareal.hs")
+            && sender == user_id!("@user:notareal.hs")
+            && unsigned.is_empty()
+    );
+}

crates/ruma-common/tests/events/mod.rs

@@ -1,5 +1,6 @@
 #![cfg(feature = "events")]
 
+mod audio;
 mod compat;
 mod enums;
 mod ephemeral_event;

crates/ruma/Cargo.toml

@@ -117,6 +117,7 @@ unstable-msc2448 = [
 unstable-msc2675 = ["ruma-common/unstable-msc2675"]
 unstable-msc2676 = ["ruma-common/unstable-msc2676"]
 unstable-msc2677 = ["ruma-common/unstable-msc2677"]
+unstable-msc3246 = ["ruma-common/unstable-msc3246"]
 unstable-msc3551 = ["ruma-common/unstable-msc3551"]
 unstable-msc3552 = ["ruma-common/unstable-msc3552"]
 unstable-msc3553 = ["ruma-common/unstable-msc3553"]
@@ -132,6 +133,7 @@ __ci = [
     "unstable-msc2675",
     "unstable-msc2676",
     "unstable-msc2677",
+    "unstable-msc3246",
     "unstable-msc3551",
     "unstable-msc3552",
     "unstable-msc3553",