diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index b5b35ed063..7730a7e517 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -616,7 +616,7 @@ jobs:
       - uses: Swatinem/rust-cache@v2
 
       - name: "Build benchmarks"
-        run: cargo codspeed build --features codspeed -p ruff_benchmark
+        run: cargo codspeed build -p ruff_benchmark
 
       - name: "Run benchmarks"
         uses: CodSpeedHQ/action@v2
diff --git a/Cargo.lock b/Cargo.lock
index c072bcac55..fc87ccc9b8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2048,7 +2048,6 @@ name = "ruff_benchmark"
 version = "0.0.0"
 dependencies = [
  "codspeed-criterion-compat",
- "criterion",
  "mimalloc",
  "once_cell",
  "red_knot",
diff --git a/Cargo.toml b/Cargo.toml
index b9aaa7acf3..8a3b382a6f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -58,7 +58,6 @@ console_error_panic_hook = { version = "0.1.7" }
 console_log = { version = "1.0.0" }
 countme = { version = "3.0.1" }
 compact_str = "0.8.0"
-criterion = { version = "0.5.1", default-features = false }
 crossbeam = { version = "0.8.4" }
 dashmap = { version = "6.0.1" }
 drop_bomb = { version = "0.1.5" }
diff --git a/crates/ruff_benchmark/Cargo.toml b/crates/ruff_benchmark/Cargo.toml
index b2be6ee580..763a622988 100644
--- a/crates/ruff_benchmark/Cargo.toml
+++ b/crates/ruff_benchmark/Cargo.toml
@@ -41,8 +41,7 @@ serde = { workspace = true }
 serde_json = { workspace = true }
 url = { workspace = true }
 ureq = { workspace = true }
-criterion = { workspace = true, default-features = false }
-codspeed-criterion-compat = { workspace = true, default-features = false, optional = true }
+codspeed-criterion-compat = { workspace = true, default-features = false }
 
 [dev-dependencies]
 ruff_db = { workspace = true }
@@ -56,9 +55,6 @@ red_knot = { workspace = true }
 [lints]
 workspace = true
 
-[features]
-codspeed = ["codspeed-criterion-compat"]
-
 [target.'cfg(target_os = "windows")'.dev-dependencies]
 mimalloc = { workspace = true }
 
diff --git a/crates/ruff_benchmark/benches/formatter.rs b/crates/ruff_benchmark/benches/formatter.rs
index af2b1caa76..740fcf1fa3 100644
--- a/crates/ruff_benchmark/benches/formatter.rs
+++ b/crates/ruff_benchmark/benches/formatter.rs
@@ -1,8 +1,9 @@
 use std::path::Path;
 
-use ruff_benchmark::criterion::{
+use codspeed_criterion_compat::{
     criterion_group, criterion_main, BenchmarkId, Criterion, Throughput,
 };
+
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
 use ruff_python_formatter::{format_module_ast, PreviewMode, PyFormatOptions};
 use ruff_python_parser::{parse, Mode};
diff --git a/crates/ruff_benchmark/benches/lexer.rs b/crates/ruff_benchmark/benches/lexer.rs
index 64b68a7a35..6e8488a552 100644
--- a/crates/ruff_benchmark/benches/lexer.rs
+++ b/crates/ruff_benchmark/benches/lexer.rs
@@ -1,6 +1,7 @@
-use ruff_benchmark::criterion::{
+use codspeed_criterion_compat::{
     criterion_group, criterion_main, measurement::WallTime, BenchmarkId, Criterion, Throughput,
 };
+
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
 use ruff_python_parser::{lexer, Mode, TokenKind};
 
diff --git a/crates/ruff_benchmark/benches/linter.rs b/crates/ruff_benchmark/benches/linter.rs
index dc27674ade..9da2437ced 100644
--- a/crates/ruff_benchmark/benches/linter.rs
+++ b/crates/ruff_benchmark/benches/linter.rs
@@ -1,6 +1,8 @@
-use ruff_benchmark::criterion::{
-    criterion_group, criterion_main, BenchmarkGroup, BenchmarkId, Criterion, Throughput,
+use codspeed_criterion_compat::{
+    self as criterion, criterion_group, criterion_main, BenchmarkGroup, BenchmarkId, Criterion,
+    Throughput,
 };
+use criterion::measurement;
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
 use ruff_linter::linter::{lint_only, ParseSource};
 use ruff_linter::rule_selector::PreviewOptions;
@@ -44,7 +46,7 @@ fn create_test_cases() -> Result<Vec<TestCase>, TestFileDownloadError> {
     ])
 }
 
-fn benchmark_linter(mut group: BenchmarkGroup, settings: &LinterSettings) {
+fn benchmark_linter(mut group: BenchmarkGroup<measurement::WallTime>, settings: &LinterSettings) {
     let test_cases = create_test_cases().unwrap();
 
     for case in test_cases {
diff --git a/crates/ruff_benchmark/benches/parser.rs b/crates/ruff_benchmark/benches/parser.rs
index ec2fa671c1..f12526d072 100644
--- a/crates/ruff_benchmark/benches/parser.rs
+++ b/crates/ruff_benchmark/benches/parser.rs
@@ -1,6 +1,7 @@
-use ruff_benchmark::criterion::{
+use codspeed_criterion_compat::{
     criterion_group, criterion_main, measurement::WallTime, BenchmarkId, Criterion, Throughput,
 };
+
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
 use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor};
 use ruff_python_ast::Stmt;
diff --git a/crates/ruff_benchmark/benches/red_knot.rs b/crates/ruff_benchmark/benches/red_knot.rs
index 0b6bdea0cc..07abdafa6b 100644
--- a/crates/ruff_benchmark/benches/red_knot.rs
+++ b/crates/ruff_benchmark/benches/red_knot.rs
@@ -1,10 +1,9 @@
 #![allow(clippy::disallowed_names)]
 
+use codspeed_criterion_compat::{criterion_group, criterion_main, BatchSize, Criterion};
+
 use red_knot::db::RootDatabase;
 use red_knot::workspace::WorkspaceMetadata;
-use ruff_benchmark::criterion::{
-    criterion_group, criterion_main, BatchSize, Criterion, Throughput,
-};
 use ruff_db::files::{system_path_to_file, vendored_path_to_file, File};
 use ruff_db::parsed::parsed_module;
 use ruff_db::program::{ProgramSettings, SearchPathSettings, TargetVersion};
@@ -100,10 +99,7 @@ fn setup_case() -> Case {
 }
 
 fn benchmark_without_parse(criterion: &mut Criterion) {
-    let mut group = criterion.benchmark_group("red_knot/check_file");
-    group.throughput(Throughput::Bytes(FOO_CODE.len() as u64));
-
-    group.bench_function("red_knot_check_file[without_parse]", |b| {
+    criterion.bench_function("red_knot_check_file[without_parse]", |b| {
         b.iter_batched_ref(
             || {
                 let case = setup_case();
@@ -123,15 +119,10 @@ fn benchmark_without_parse(criterion: &mut Criterion) {
             BatchSize::SmallInput,
         );
     });
-
-    group.finish();
 }
 
 fn benchmark_incremental(criterion: &mut Criterion) {
-    let mut group = criterion.benchmark_group("red_knot/check_file");
-    group.throughput(Throughput::Bytes(FOO_CODE.len() as u64));
-
-    group.bench_function("red_knot_check_file[incremental]", |b| {
+    criterion.bench_function("red_knot_check_file[incremental]", |b| {
         b.iter_batched_ref(
             || {
                 let mut case = setup_case();
@@ -156,15 +147,10 @@ fn benchmark_incremental(criterion: &mut Criterion) {
             BatchSize::SmallInput,
         );
     });
-
-    group.finish();
 }
 
 fn benchmark_cold(criterion: &mut Criterion) {
-    let mut group = criterion.benchmark_group("red_knot/check_file");
-    group.throughput(Throughput::Bytes(FOO_CODE.len() as u64));
-
-    group.bench_function("red_knot_check_file[cold]", |b| {
+    criterion.bench_function("red_knot_check_file[cold]", |b| {
         b.iter_batched_ref(
             setup_case,
             |case| {
@@ -176,11 +162,12 @@ fn benchmark_cold(criterion: &mut Criterion) {
             BatchSize::SmallInput,
         );
     });
-
-    group.finish();
 }
 
-criterion_group!(cold, benchmark_cold);
-criterion_group!(without_parse, benchmark_without_parse);
-criterion_group!(incremental, benchmark_incremental);
-criterion_main!(without_parse, cold, incremental);
+criterion_group!(
+    check_file,
+    benchmark_cold,
+    benchmark_without_parse,
+    benchmark_incremental
+);
+criterion_main!(check_file);
diff --git a/crates/ruff_benchmark/src/criterion.rs b/crates/ruff_benchmark/src/criterion.rs
deleted file mode 100644
index 213403c1ce..0000000000
--- a/crates/ruff_benchmark/src/criterion.rs
+++ /dev/null
@@ -1,11 +0,0 @@
-//! This module re-exports the criterion API but picks the right backend depending on whether
-//! the benchmarks are built to run locally or with codspeed
-
-#[cfg(not(codspeed))]
-pub use criterion::*;
-
-#[cfg(not(codspeed))]
-pub type BenchmarkGroup<'a> = criterion::BenchmarkGroup<'a, measurement::WallTime>;
-
-#[cfg(codspeed)]
-pub use codspeed_criterion_compat::*;
diff --git a/crates/ruff_benchmark/src/lib.rs b/crates/ruff_benchmark/src/lib.rs
index b5e60cd3b4..70d2e7a34f 100644
--- a/crates/ruff_benchmark/src/lib.rs
+++ b/crates/ruff_benchmark/src/lib.rs
@@ -1,5 +1,3 @@
-pub mod criterion;
-
 use std::fmt::{Display, Formatter};
 use std::path::PathBuf;
 use std::process::Command;