diff --git a/Cargo.lock b/Cargo.lock
index 36bae6ff2c..f4326c3ffa 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2336,6 +2336,7 @@ dependencies = [
  "ruff_python_formatter",
  "ruff_python_parser",
  "ruff_python_trivia",
+ "rustc-hash 2.0.0",
  "serde",
  "serde_json",
  "tikv-jemallocator",
diff --git a/crates/ruff_benchmark/Cargo.toml b/crates/ruff_benchmark/Cargo.toml
index 9df32cd5ee..05328759cd 100644
--- a/crates/ruff_benchmark/Cargo.toml
+++ b/crates/ruff_benchmark/Cargo.toml
@@ -41,6 +41,7 @@ codspeed-criterion-compat = { workspace = true, default-features = false, option
 criterion = { workspace = true, default-features = false }
 once_cell = { workspace = true }
 rayon = { workspace = true }
+rustc-hash = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 url = { workspace = true }
diff --git a/crates/ruff_benchmark/benches/red_knot.rs b/crates/ruff_benchmark/benches/red_knot.rs
index d2a52bcd96..f0a577cbd2 100644
--- a/crates/ruff_benchmark/benches/red_knot.rs
+++ b/crates/ruff_benchmark/benches/red_knot.rs
@@ -10,19 +10,21 @@ use ruff_benchmark::criterion::{criterion_group, criterion_main, BatchSize, Crit
 use ruff_benchmark::TestFile;
 use ruff_db::files::{system_path_to_file, File};
 use ruff_db::source::source_text;
-use ruff_db::system::{MemoryFileSystem, SystemPath, TestSystem};
+use ruff_db::system::{MemoryFileSystem, SystemPath, SystemPathBuf, TestSystem};
+use rustc_hash::FxHashSet;
 
 struct Case {
     db: RootDatabase,
     fs: MemoryFileSystem,
     re: File,
-    re_path: &'static SystemPath,
+    re_path: SystemPathBuf,
 }
 
 const TOMLLIB_312_URL: &str = "https://raw.githubusercontent.com/python/cpython/8e8a4baf652f6e1cee7acde9d78c4b6154539748/Lib/tomllib";
 
 // The failed import from 'collections.abc' is due to lack of support for 'import *'.
 static EXPECTED_DIAGNOSTICS: &[&str] = &[
+    "/src/tomllib/__init__.py:10:30: Name '__name__' used when not defined.",
     "/src/tomllib/_parser.py:7:29: Module 'collections.abc' has no member 'Iterable'",
     "Line 69 is too long (89 characters)",
     "Use double quotes for strings",
@@ -40,23 +42,21 @@ fn get_test_file(name: &str) -> TestFile {
     TestFile::try_download(&path, &url).unwrap()
 }
 
+fn tomllib_path(filename: &str) -> SystemPathBuf {
+    SystemPathBuf::from(format!("/src/tomllib/{filename}").as_str())
+}
+
 fn setup_case() -> Case {
     let system = TestSystem::default();
     let fs = system.memory_file_system().clone();
-    let parser_path = SystemPath::new("/src/tomllib/_parser.py");
-    let re_path = SystemPath::new("/src/tomllib/_re.py");
-    fs.write_files([
+
+    let tomllib_filenames = ["__init__.py", "_parser.py", "_re.py", "_types.py"];
+    fs.write_files(tomllib_filenames.iter().map(|filename| {
         (
-            SystemPath::new("/src/tomllib/__init__.py"),
-            get_test_file("__init__.py").code(),
-        ),
-        (parser_path, get_test_file("_parser.py").code()),
-        (re_path, get_test_file("_re.py").code()),
-        (
-            SystemPath::new("/src/tomllib/_types.py"),
-            get_test_file("_types.py").code(),
-        ),
-    ])
+            tomllib_path(filename),
+            get_test_file(filename).code().to_string(),
+        )
+    }))
     .unwrap();
 
     let src_root = SystemPath::new("/src");
@@ -71,12 +71,15 @@ fn setup_case() -> Case {
     .unwrap();
 
     let mut db = RootDatabase::new(metadata, system).unwrap();
-    let parser = system_path_to_file(&db, parser_path).unwrap();
 
-    db.workspace().open_file(&mut db, parser);
-
-    let re = system_path_to_file(&db, re_path).unwrap();
+    let tomllib_files: FxHashSet<File> = tomllib_filenames
+        .iter()
+        .map(|filename| system_path_to_file(&db, tomllib_path(filename)).unwrap())
+        .collect();
+    db.workspace().set_open_files(&mut db, tomllib_files);
 
+    let re_path = tomllib_path("_re.py");
+    let re = system_path_to_file(&db, &re_path).unwrap();
     Case {
         db,
         fs,
@@ -112,7 +115,7 @@ fn benchmark_incremental(criterion: &mut Criterion) {
 
                 case.fs
                     .write_file(
-                        case.re_path,
+                        &case.re_path,
                         format!("{}\n# A comment\n", source_text(&case.db, case.re).as_str()),
                     )
                     .unwrap();
@@ -124,7 +127,7 @@ fn benchmark_incremental(criterion: &mut Criterion) {
 
                 db.apply_changes(
                     vec![ChangeEvent::Changed {
-                        path: case.re_path.to_path_buf(),
+                        path: case.re_path.clone(),
                         kind: ChangedKind::FileContent,
                     }],
                     None,