diff --git a/crates/ruma-common/Cargo.toml b/crates/ruma-common/Cargo.toml
index 89706297..58b00d33 100644
--- a/crates/ruma-common/Cargo.toml
+++ b/crates/ruma-common/Cargo.toml
@@ -52,7 +52,6 @@ unstable-msc3700 = []
 [dependencies]
 base64 = "0.13.0"
 bytes = "1.0.1"
-criterion = { version = "0.3.3", optional = true }
 form_urlencoded = "1.0.0"
 getrandom = { version = "0.2.6", optional = true }
 http = { version = "0.2.2", optional = true }
@@ -73,6 +72,9 @@ url = "2.2.2"
 uuid = { version = "1.0.0", optional = true, features = ["v4"] }
 wildmatch = "2.0.0"
 
+# dev-dependencies can't be optional, so this is a regular dependency
+criterion = { version = "0.3.3", optional = true }
+
 [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies]
 js-sys = { version = "0.3", optional = true }
 
@@ -86,3 +88,4 @@ trybuild = "1.0.42"
 [[bench]]
 name = "event_deserialize"
 harness = false
+required-features = ["criterion"]
diff --git a/crates/ruma-common/benches/event_deserialize.rs b/crates/ruma-common/benches/event_deserialize.rs
index 9d2e558d..a0139e55 100644
--- a/crates/ruma-common/benches/event_deserialize.rs
+++ b/crates/ruma-common/benches/event_deserialize.rs
@@ -7,7 +7,6 @@
 #![cfg(feature = "events")]
 #![allow(unused_imports, dead_code)]
 
-#[cfg(feature = "criterion")]
 use criterion::{criterion_group, criterion_main, Criterion};
 use ruma_common::{
     events::{
@@ -51,7 +50,6 @@ fn power_levels() -> serde_json::Value {
     })
 }
 
-#[cfg(feature = "criterion")]
 fn deserialize_any_room_event(c: &mut Criterion) {
     let json_data = power_levels();
 
@@ -62,7 +60,6 @@ fn deserialize_any_room_event(c: &mut Criterion) {
     });
 }
 
-#[cfg(feature = "criterion")]
 fn deserialize_any_state_event(c: &mut Criterion) {
     let json_data = power_levels();
 
@@ -73,7 +70,6 @@ fn deserialize_any_state_event(c: &mut Criterion) {
     });
 }
 
-#[cfg(feature = "criterion")]
 fn deserialize_specific_event(c: &mut Criterion) {
     let json_data = power_levels();
 
@@ -87,7 +83,6 @@ fn deserialize_specific_event(c: &mut Criterion) {
     });
 }
 
-#[cfg(feature = "criterion")]
 criterion_group!(
     benches,
     deserialize_any_room_event,
@@ -95,12 +90,4 @@ criterion_group!(
     deserialize_specific_event
 );
 
-#[cfg(feature = "criterion")]
 criterion_main!(benches);
-
-#[cfg(not(feature = "criterion"))]
-fn main() {
-    // To run the benchmarks the "criterion" feature must be enabled use:
-    // `cargo bench --features criterion --bench event_deserialize`
-    panic!("Enable the criterion feature to run benchmarks");
-}
diff --git a/crates/ruma-state-res/Cargo.toml b/crates/ruma-state-res/Cargo.toml
index ed0e67b7..8933947a 100644
--- a/crates/ruma-state-res/Cargo.toml
+++ b/crates/ruma-state-res/Cargo.toml
@@ -27,8 +27,10 @@ serde_json = "1.0.60"
 thiserror = "1.0.26"
 tracing = "0.1.26"
 
+# dev-dependencies can't be optional, so this is a regular dependency
+criterion = { version = "0.3.3", optional = true }
+
 [dev-dependencies]
-criterion = "0.3.3"
 maplit = "1.0.2"
 rand = "0.8.3"
 ruma-common = { version = "0.9.2", path = "../ruma-common", features = ["unstable-pdu"] }
@@ -37,3 +39,4 @@ tracing-subscriber = "0.3.3"
 [[bench]]
 name = "state_res_bench"
 harness = false
+required-features = ["criterion"]