diff --git a/ruma-events/Cargo.toml b/ruma-events/Cargo.toml
index ef4dbb45..6372cb1e 100644
--- a/ruma-events/Cargo.toml
+++ b/ruma-events/Cargo.toml
@@ -13,6 +13,7 @@ version = "0.21.3"
 edition = "2018"
 
 [dependencies]
+criterion = { version = "0.3.2", optional = true }
 js_int = { version = "0.1.7", features = ["serde"] }
 ruma-common = { version = "0.1.3", path = "../ruma-common" }
 ruma-events-macros = { version = "=0.21.3", path = "../ruma-events-macros" }
@@ -27,7 +28,6 @@ maplit = "1.0.2"
 matches = "0.1.8"
 ruma-identifiers = { version = "0.16.2", path = "../ruma-identifiers", features = ["rand"] }
 trybuild = "1.0.30"
-criterion = "0.3.2"
 
 [[bench]]
 name = "event_deserialize"
diff --git a/ruma-events/benches/event_deserialize.rs b/ruma-events/benches/event_deserialize.rs
index 47026ade..997103a2 100644
--- a/ruma-events/benches/event_deserialize.rs
+++ b/ruma-events/benches/event_deserialize.rs
@@ -2,8 +2,11 @@
 // or pass any other args to it, it fails with the error
 // `cargo bench unknown option --save-baseline`.
 // To pass args to criterion, use this form
-// `cargo bench --bench <name of the bench> -- --save-baseline <name>`.
+// `cargo bench --features criterion --bench <name of the bench> -- --save-baseline <name>`.
 
+#![allow(unused_imports, dead_code)]
+
+#[cfg(feature = "criterion")]
 use criterion::{criterion_group, criterion_main, Criterion};
 use ruma_events::{
     room::power_levels::PowerLevelsEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, EventJson,
@@ -44,6 +47,7 @@ fn power_levels() -> serde_json::Value {
     })
 }
 
+#[cfg(feature = "criterion")]
 fn deserialize_any_event(c: &mut Criterion) {
     let json_data = power_levels();
 
@@ -57,6 +61,7 @@ fn deserialize_any_event(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "criterion")]
 fn deserialize_any_room_event(c: &mut Criterion) {
     let json_data = power_levels();
 
@@ -70,6 +75,7 @@ fn deserialize_any_room_event(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "criterion")]
 fn deserialize_any_state_event(c: &mut Criterion) {
     let json_data = power_levels();
 
@@ -83,6 +89,7 @@ fn deserialize_any_state_event(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "criterion")]
 fn deserialize_specific_event(c: &mut Criterion) {
     let json_data = power_levels();
 
@@ -98,6 +105,7 @@ fn deserialize_specific_event(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "criterion")]
 criterion_group!(
     benches,
     deserialize_any_event,
@@ -106,4 +114,12 @@ criterion_group!(
     deserialize_specific_event
 );
 
+#[cfg(feature = "criterion")]
 criterion_main!(benches);
+
+#[cfg(not(feature = "criterion"))]
+fn main() {
+    // To run the benchmarks the "criterion" feature must be enabled use:
+    // `cargo bench --features criterion --bench event_deserialize`
+    panic!("Enable the criterion feature to run benchmarks");
+}