diff --git a/Cargo.lock b/Cargo.lock
index 3416ce55e7..d77e0630b2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2322,6 +2322,7 @@ name = "red_knot_test"
 version = "0.0.0"
 dependencies = [
  "anyhow",
+ "camino",
  "colored",
  "memchr",
  "red_knot_python_semantic",
diff --git a/crates/red_knot_python_semantic/tests/mdtest.rs b/crates/red_knot_python_semantic/tests/mdtest.rs
index dac4175667..9a56c11d9a 100644
--- a/crates/red_knot_python_semantic/tests/mdtest.rs
+++ b/crates/red_knot_python_semantic/tests/mdtest.rs
@@ -1,7 +1,6 @@
-use std::ffi::OsStr;
-use std::path::Path;
-
+use camino::Utf8Path;
 use dir_test::{dir_test, Fixture};
+use std::path::Path;
 
 /// See `crates/red_knot_test/README.md` for documentation on these tests.
 #[dir_test(
@@ -10,16 +9,46 @@ use dir_test::{dir_test, Fixture};
 )]
 #[allow(clippy::needless_pass_by_value)]
 fn mdtest(fixture: Fixture<&str>) {
-    let fixture_path = Path::new(fixture.path());
+    let fixture_path = Utf8Path::new(fixture.path());
     let crate_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
-    let workspace_root = crate_dir.parent().and_then(Path::parent).unwrap();
+    let workspace_root = crate_dir.ancestors().nth(2).unwrap();
 
-    let long_title = fixture_path
-        .strip_prefix(workspace_root)
-        .unwrap()
-        .to_str()
-        .unwrap();
-    let short_title = fixture_path.file_name().and_then(OsStr::to_str).unwrap();
+    let long_title = fixture_path.strip_prefix(workspace_root).unwrap();
+    let short_title = fixture_path.file_name().unwrap();
 
-    red_knot_test::run(fixture_path, long_title, short_title);
+    let test_name = test_name("mdtest", fixture_path);
+
+    red_knot_test::run(fixture_path, long_title.as_str(), short_title, &test_name);
+}
+
+/// Constructs the test name used for individual markdown files
+///
+/// This code is copied from <https://github.com/fe-lang/dir-test/blob/1c0f41c480a3490bc2653a043ff6e3f8085a1f47/macros/src/lib.rs#L104-L138>
+/// and should be updated if they diverge
+fn test_name(test_func_name: &str, fixture_path: &Utf8Path) -> String {
+    assert!(fixture_path.is_file());
+
+    let dir_path = format!("{}/resources/mdtest", std::env!("CARGO_MANIFEST_DIR"));
+    let rel_path = fixture_path.strip_prefix(dir_path).unwrap();
+    assert!(rel_path.is_relative());
+
+    let mut test_name = test_func_name.to_owned();
+    test_name.push_str("__");
+
+    for component in rel_path.parent().unwrap().components() {
+        let component = component
+            .as_str()
+            .replace(|c: char| c.is_ascii_punctuation(), "_");
+        test_name.push_str(&component);
+        test_name.push('_');
+    }
+
+    test_name.push_str(
+        &rel_path
+            .file_stem()
+            .unwrap()
+            .replace(|c: char| c.is_ascii_punctuation(), "_"),
+    );
+
+    test_name
 }
diff --git a/crates/red_knot_test/Cargo.toml b/crates/red_knot_test/Cargo.toml
index 68c7f13e96..fbfef2d4db 100644
--- a/crates/red_knot_test/Cargo.toml
+++ b/crates/red_knot_test/Cargo.toml
@@ -20,6 +20,7 @@ ruff_source_file = { workspace = true }
 ruff_text_size = { workspace = true }
 
 anyhow = { workspace = true }
+camino = { workspace = true }
 colored = { workspace = true }
 memchr = { workspace = true }
 regex = { workspace = true }
diff --git a/crates/red_knot_test/src/lib.rs b/crates/red_knot_test/src/lib.rs
index 62c3c50a76..23957a9dea 100644
--- a/crates/red_knot_test/src/lib.rs
+++ b/crates/red_knot_test/src/lib.rs
@@ -1,3 +1,4 @@
+use camino::Utf8Path;
 use colored::Colorize;
 use parser as test_parser;
 use red_knot_python_semantic::types::check_types;
@@ -7,7 +8,6 @@ use ruff_db::parsed::parsed_module;
 use ruff_db::system::{DbWithTestSystem, SystemPathBuf};
 use ruff_source_file::LineIndex;
 use ruff_text_size::TextSize;
-use std::path::Path;
 
 mod assertion;
 mod db;
@@ -21,12 +21,12 @@ const MDTEST_TEST_FILTER: &str = "MDTEST_TEST_FILTER";
 ///
 /// Panic on test failure, and print failure details.
 #[allow(clippy::print_stdout)]
-pub fn run(path: &Path, long_title: &str, short_title: &str) {
+pub fn run(path: &Utf8Path, long_title: &str, short_title: &str, test_name: &str) {
     let source = std::fs::read_to_string(path).unwrap();
     let suite = match test_parser::parse(short_title, &source) {
         Ok(suite) => suite,
         Err(err) => {
-            panic!("Error parsing `{}`: {err}", path.to_str().unwrap())
+            panic!("Error parsing `{path}`: {err}")
         }
     };
 
@@ -67,7 +67,7 @@ pub fn run(path: &Path, long_title: &str, short_title: &str) {
                 test.name()
             );
             println!(
-                "{MDTEST_TEST_FILTER}=\"{}\" cargo test -p red_knot_python_semantic --test mdtest",
+                "{MDTEST_TEST_FILTER}=\"{}\" cargo test -p red_knot_python_semantic --test mdtest -- {test_name}",
                 test.name()
             );
         }