Shard ty walltime benchmarks (#20791)

.github/workflows/ci.yaml

@@ -995,12 +995,16 @@ jobs:
           token: ${{ secrets.CODSPEED_TOKEN }}
 
   benchmarks-walltime:
+    name: "benchmarks walltime (${{ matrix.benchmarks }})"
     runs-on: codspeed-macro
     needs: determine_changes
     if: ${{ github.repository == 'astral-sh/ruff' && !contains(github.event.pull_request.labels.*.name, 'no-test') && (needs.determine_changes.outputs.ty == 'true' || github.ref == 'refs/heads/main') }}
     timeout-minutes: 20
-    env:
+    strategy:
-      TY_LOG: ruff_benchmark=debug
+      matrix:
+        benchmarks:
+          - "medium|multithreaded"
+          - "small|large"
     steps:
       - name: "Checkout Branch"
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -1030,5 +1034,5 @@ jobs:
           CODSPEED_PERF_ENABLED: false
         with:
           mode: walltime
-          run: cargo codspeed run
+          run: cargo codspeed run --bench ty_walltime "${{ matrix.benchmarks }}"
           token: ${{ secrets.CODSPEED_TOKEN }}

Cargo.lock

@@ -214,15 +214,6 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
-[[package]]
-name = "bincode"
-version = "1.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "bincode"
 version = "2.0.1"
@@ -243,6 +234,26 @@ dependencies = [
  "virtue",
 ]
 
+[[package]]
+name = "bindgen"
+version = "0.72.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
+dependencies = [
+ "bitflags 2.9.4",
+ "cexpr",
+ "clang-sys",
+ "itertools 0.13.0",
+ "log",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+ "syn",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -350,6 +361,15 @@ dependencies = [
  "shlex",
 ]
 
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "cfg-if"
 version = "1.0.3"
@@ -400,6 +420,17 @@ dependencies = [
  "half",
 ]
 
+[[package]]
+name = "clang-sys"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
 [[package]]
 name = "clap"
 version = "4.5.48"
@@ -486,16 +517,17 @@ dependencies = [
 
 [[package]]
 name = "codspeed"
-version = "3.0.5"
+version = "4.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35584c5fcba8059780748866387fb97c5a203bcfc563fc3d0790af406727a117"
+checksum = "d0f62ea8934802f8b374bf691eea524c3aa444d7014f604dd4182a3667b69510"
 dependencies = [
  "anyhow",
- "bincode 1.3.3",
+ "bindgen",
+ "cc",
  "colored 2.2.0",
  "glob",
  "libc",
- "nix 0.29.0",
+ "nix 0.30.1",
  "serde",
  "serde_json",
  "statrs",
@@ -504,20 +536,22 @@ dependencies = [
 
 [[package]]
 name = "codspeed-criterion-compat"
-version = "3.0.5"
+version = "4.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78f6c1c6bed5fd84d319e8b0889da051daa361c79b7709c9394dfe1a882bba67"
+checksum = "d87efbc015fc0ff1b2001cd87df01c442824de677e01a77230bf091534687abb"
 dependencies = [
+ "clap",
  "codspeed",
  "codspeed-criterion-compat-walltime",
  "colored 2.2.0",
+ "regex",
 ]
 
 [[package]]
 name = "codspeed-criterion-compat-walltime"
-version = "3.0.5"
+version = "4.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c989289ce6b1cbde72ed560496cb8fbf5aa14d5ef5666f168e7f87751038352e"
+checksum = "ae5713ace440123bb4f1f78dd068d46872cb8548bfe61f752e7b2ad2c06d7f00"
 dependencies = [
  "anes",
  "cast",
@@ -540,20 +574,22 @@ dependencies = [
 
 [[package]]
 name = "codspeed-divan-compat"
-version = "3.0.5"
+version = "4.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adf64eda57508448d59efd940bad62ede7c50b0d451a150b8d6a0eca642792a6"
+checksum = "95b4214b974f8f5206497153e89db90274e623f06b00bf4b9143eeb7735d975d"
 dependencies = [
+ "clap",
  "codspeed",
  "codspeed-divan-compat-macros",
  "codspeed-divan-compat-walltime",
+ "regex",
 ]
 
 [[package]]
 name = "codspeed-divan-compat-macros"
-version = "3.0.5"
+version = "4.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "058167258e819b16a4ba601fdfe270349ef191154758dbce122c62a698f70ba8"
+checksum = "a53f34a16cb70ce4fd9ad57e1db016f0718e434f34179ca652006443b9a39967"
 dependencies = [
  "divan-macros",
  "itertools 0.14.0",
@@ -565,9 +601,9 @@ dependencies = [
 
 [[package]]
 name = "codspeed-divan-compat-walltime"
-version = "3.0.5"
+version = "4.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48f9866ee3a4ef9d2868823ea5811886763af244f2df584ca247f49281c43f1f"
+checksum = "e8a5099050c8948dce488b8eaa2e68dc5cf571cb8f9fce99aaaecbdddb940bcd"
 dependencies = [
  "cfg-if",
  "clap",
@@ -1830,6 +1866,16 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "libloading"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
+dependencies = [
+ "cfg-if",
+ "windows-link 0.2.0",
+]
+
 [[package]]
 name = "libmimalloc-sys"
 version = "0.1.44"
@@ -2482,6 +2528,16 @@ dependencies = [
  "yansi",
 ]
 
+[[package]]
+name = "prettyplease"
+version = "0.2.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
+dependencies = [
+ "proc-macro2",
+ "syn",
+]
+
 [[package]]
 name = "proc-macro-crate"
 version = "3.4.0"
@@ -2713,9 +2769,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.11.2"
+version = "1.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912"
+checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -2725,9 +2781,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.10"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6"
+checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -2764,7 +2820,7 @@ dependencies = [
  "anyhow",
  "argfile",
  "assert_fs",
- "bincode 2.0.1",
+ "bincode",
  "bitflags 2.9.4",
  "cachedir",
  "clap",

Cargo.toml

@@ -71,8 +71,8 @@ clap = { version = "4.5.3", features = ["derive"] }
 clap_complete_command = { version = "0.6.0" }
 clearscreen = { version = "4.0.0" }
 csv = { version = "1.3.1" }
-divan = { package = "codspeed-divan-compat", version = "3.0.2" }
+divan = { package = "codspeed-divan-compat", version = "4.0.4" }
-codspeed-criterion-compat = { version = "3.0.2", default-features = false }
+codspeed-criterion-compat = { version = "4.0.4", default-features = false }
 colored = { version = "3.0.0" }
 console_error_panic_hook = { version = "0.1.7" }
 console_log = { version = "1.0.0" }

crates/ruff_benchmark/benches/ty.rs

@@ -599,7 +599,7 @@ impl<'a> ProjectBenchmark<'a> {
             self.project
                 .check_paths()
                 .iter()
-                .map(|path| path.to_path_buf())
+                .map(|path| SystemPathBuf::from(*path))
                 .collect(),
         );
 
@@ -645,8 +645,8 @@ fn hydra(criterion: &mut Criterion) {
             name: "hydra-zen",
             repository: "https://github.com/mit-ll-responsible-ai/hydra-zen",
             commit: "dd2b50a9614c6f8c46c5866f283c8f7e7a960aa8",
-            paths: vec![SystemPath::new("src")],
+            paths: &["src"],
-            dependencies: vec!["pydantic", "beartype", "hydra-core"],
+            dependencies: &["pydantic", "beartype", "hydra-core"],
             max_dep_date: "2025-06-17",
             python_version: PythonVersion::PY313,
         },
@@ -662,8 +662,8 @@ fn attrs(criterion: &mut Criterion) {
             name: "attrs",
             repository: "https://github.com/python-attrs/attrs",
             commit: "a6ae894aad9bc09edc7cdad8c416898784ceec9b",
-            paths: vec![SystemPath::new("src")],
+            paths: &["src"],
-            dependencies: vec![],
+            dependencies: &[],
             max_dep_date: "2025-06-17",
             python_version: PythonVersion::PY313,
         },
@@ -679,8 +679,8 @@ fn anyio(criterion: &mut Criterion) {
             name: "anyio",
             repository: "https://github.com/agronholm/anyio",
             commit: "561d81270a12f7c6bbafb5bc5fad99a2a13f96be",
-            paths: vec![SystemPath::new("src")],
+            paths: &["src"],
-            dependencies: vec![],
+            dependencies: &[],
             max_dep_date: "2025-06-17",
             python_version: PythonVersion::PY313,
         },
@@ -696,8 +696,8 @@ fn datetype(criterion: &mut Criterion) {
             name: "DateType",
             repository: "https://github.com/glyph/DateType",
             commit: "57c9c93cf2468069f72945fc04bf27b64100dad8",
-            paths: vec![SystemPath::new("src")],
+            paths: &["src"],
-            dependencies: vec![],
+            dependencies: &[],
             max_dep_date: "2025-07-04",
             python_version: PythonVersion::PY313,
         },

crates/ruff_benchmark/benches/ty_walltime.rs

@@ -1,6 +1,5 @@
-use std::fmt::{Display, Formatter};
-
 use divan::{Bencher, bench};
+use std::fmt::{Display, Formatter};
 
 use rayon::ThreadPoolBuilder;
 use ruff_benchmark::real_world_projects::{InstalledProject, RealWorldProject};
@@ -13,29 +12,39 @@ use ty_project::metadata::value::{RangedValue, RelativePathBuf};
 use ty_project::{Db, ProjectDatabase, ProjectMetadata};
 
 struct Benchmark<'a> {
-    project: InstalledProject<'a>,
+    project: RealWorldProject<'a>,
+    installed_project: std::sync::OnceLock<InstalledProject<'a>>,
     max_diagnostics: usize,
 }
 
 impl<'a> Benchmark<'a> {
-    fn new(project: RealWorldProject<'a>, max_diagnostics: usize) -> Self {
+    const fn new(project: RealWorldProject<'a>, max_diagnostics: usize) -> Self {
-        let setup_project = project.setup().expect("Failed to setup project");
-
         Self {
-            project: setup_project,
+            project,
+            installed_project: std::sync::OnceLock::new(),
             max_diagnostics,
         }
     }
 
+    fn installed_project(&self) -> &InstalledProject<'a> {
+        self.installed_project.get_or_init(|| {
+            self.project
+                .clone()
+                .setup()
+                .expect("Failed to setup project")
+        })
+    }
+
     fn setup_iteration(&self) -> ProjectDatabase {
-        let root = SystemPathBuf::from_path_buf(self.project.path.clone()).unwrap();
+        let installed_project = self.installed_project();
+        let root = SystemPathBuf::from_path_buf(installed_project.path.clone()).unwrap();
         let system = OsSystem::new(&root);
 
         let mut metadata = ProjectMetadata::discover(&root, &system).unwrap();
 
         metadata.apply_options(Options {
             environment: Some(EnvironmentOptions {
-                python_version: Some(RangedValue::cli(self.project.config.python_version)),
+                python_version: Some(RangedValue::cli(installed_project.config.python_version)),
                 python: Some(RelativePathBuf::cli(SystemPath::new(".venv"))),
                 ..EnvironmentOptions::default()
             }),
@@ -46,7 +55,7 @@ impl<'a> Benchmark<'a> {
 
         db.project().set_included_paths(
             &mut db,
-            self.project
+            installed_project
                 .check_paths()
                 .iter()
                 .map(|path| SystemPath::absolute(path, &root))
@@ -58,7 +67,7 @@ impl<'a> Benchmark<'a> {
 
 impl Display for Benchmark<'_> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.write_str(self.project.config.name)
+        self.project.name.fmt(f)
     }
 }
 
@@ -75,14 +84,13 @@ fn check_project(db: &ProjectDatabase, max_diagnostics: usize) {
     );
 }
 
-static ALTAIR: std::sync::LazyLock<Benchmark<'static>> = std::sync::LazyLock::new(|| {
+static ALTAIR: Benchmark = Benchmark::new(
-    Benchmark::new(
     RealWorldProject {
         name: "altair",
         repository: "https://github.com/vega/altair",
         commit: "d1f4a1ef89006e5f6752ef1f6df4b7a509336fba",
-            paths: vec![SystemPath::new("altair")],
+        paths: &["altair"],
-            dependencies: vec![
+        dependencies: &[
             "jinja2",
             "narwhals",
             "numpy",
@@ -97,17 +105,15 @@ static ALTAIR: std::sync::LazyLock<Benchmark<'static>> = std::sync::LazyLock::ne
         python_version: PythonVersion::PY312,
     },
     1000,
-    )
+);
-});
 
-static COLOUR_SCIENCE: std::sync::LazyLock<Benchmark<'static>> = std::sync::LazyLock::new(|| {
+static COLOUR_SCIENCE: Benchmark = Benchmark::new(
-    Benchmark::new(
     RealWorldProject {
         name: "colour-science",
         repository: "https://github.com/colour-science/colour",
         commit: "a17e2335c29e7b6f08080aa4c93cfa9b61f84757",
-            paths: vec![SystemPath::new("colour")],
+        paths: &["colour"],
-            dependencies: vec![
+        dependencies: &[
             "matplotlib",
             "numpy",
             "pandas-stubs",
@@ -118,17 +124,15 @@ static COLOUR_SCIENCE: std::sync::LazyLock<Benchmark<'static>> = std::sync::Lazy
         python_version: PythonVersion::PY310,
     },
     600,
-    )
+);
-});
 
-static FREQTRADE: std::sync::LazyLock<Benchmark<'static>> = std::sync::LazyLock::new(|| {
+static FREQTRADE: Benchmark = Benchmark::new(
-    Benchmark::new(
     RealWorldProject {
         name: "freqtrade",
         repository: "https://github.com/freqtrade/freqtrade",
         commit: "2d842ea129e56575852ee0c45383c8c3f706be19",
-            paths: vec![SystemPath::new("freqtrade")],
+        paths: &["freqtrade"],
-            dependencies: vec![
+        dependencies: &[
             "numpy",
             "pandas-stubs",
             "pydantic",
@@ -143,17 +147,15 @@ static FREQTRADE: std::sync::LazyLock<Benchmark<'static>> = std::sync::LazyLock:
         python_version: PythonVersion::PY312,
     },
     400,
-    )
+);
-});
 
-static PANDAS: std::sync::LazyLock<Benchmark<'static>> = std::sync::LazyLock::new(|| {
+static PANDAS: Benchmark = Benchmark::new(
-    Benchmark::new(
     RealWorldProject {
         name: "pandas",
         repository: "https://github.com/pandas-dev/pandas",
         commit: "5909621e2267eb67943a95ef5e895e8484c53432",
-            paths: vec![SystemPath::new("pandas")],
+        paths: &["pandas"],
-            dependencies: vec![
+        dependencies: &[
             "numpy",
             "types-python-dateutil",
             "types-pytz",
@@ -165,17 +167,15 @@ static PANDAS: std::sync::LazyLock<Benchmark<'static>> = std::sync::LazyLock::ne
         python_version: PythonVersion::PY312,
     },
     3000,
-    )
+);
-});
 
-static PYDANTIC: std::sync::LazyLock<Benchmark<'static>> = std::sync::LazyLock::new(|| {
+static PYDANTIC: Benchmark = Benchmark::new(
-    Benchmark::new(
     RealWorldProject {
         name: "pydantic",
         repository: "https://github.com/pydantic/pydantic",
         commit: "0c4a22b64b23dfad27387750cf07487efc45eb05",
-            paths: vec![SystemPath::new("pydantic")],
+        paths: &["pydantic"],
-            dependencies: vec![
+        dependencies: &[
             "annotated-types",
             "pydantic-core",
             "typing-extensions",
@@ -185,56 +185,49 @@ static PYDANTIC: std::sync::LazyLock<Benchmark<'static>> = std::sync::LazyLock::
         python_version: PythonVersion::PY39,
     },
     1000,
-    )
+);
-});
 
-static SYMPY: std::sync::LazyLock<Benchmark<'static>> = std::sync::LazyLock::new(|| {
+static SYMPY: Benchmark = Benchmark::new(
-    Benchmark::new(
     RealWorldProject {
         name: "sympy",
         repository: "https://github.com/sympy/sympy",
         commit: "22fc107a94eaabc4f6eb31470b39db65abb7a394",
-            paths: vec![SystemPath::new("sympy")],
+        paths: &["sympy"],
-            dependencies: vec!["mpmath"],
+        dependencies: &["mpmath"],
         max_dep_date: "2025-06-17",
         python_version: PythonVersion::PY312,
     },
     13000,
-    )
+);
-});
 
-static TANJUN: std::sync::LazyLock<Benchmark<'static>> = std::sync::LazyLock::new(|| {
+static TANJUN: Benchmark = Benchmark::new(
-    Benchmark::new(
     RealWorldProject {
         name: "tanjun",
         repository: "https://github.com/FasterSpeeding/Tanjun",
         commit: "69f40db188196bc59516b6c69849c2d85fbc2f4a",
-            paths: vec![SystemPath::new("tanjun")],
+        paths: &["tanjun"],
-            dependencies: vec!["hikari", "alluka"],
+        dependencies: &["hikari", "alluka"],
         max_dep_date: "2025-06-17",
         python_version: PythonVersion::PY312,
     },
     100,
-    )
+);
-});
 
-static STATIC_FRAME: std::sync::LazyLock<Benchmark<'static>> = std::sync::LazyLock::new(|| {
+static STATIC_FRAME: Benchmark = Benchmark::new(
-    Benchmark::new(
     RealWorldProject {
         name: "static-frame",
         repository: "https://github.com/static-frame/static-frame",
         commit: "34962b41baca5e7f98f5a758d530bff02748a421",
-            paths: vec![SystemPath::new("static_frame")],
+        paths: &["static_frame"],
         // N.B. `arraykit` is installed as a dependency during mypy_primer runs,
         // but it takes much longer to be installed in a Codspeed run than it does in a mypy_primer run
         // (seems to be built from source on the Codspeed CI runners for some reason).
-            dependencies: vec!["numpy"],
+        dependencies: &["numpy"],
         max_dep_date: "2025-08-09",
         python_version: PythonVersion::PY311,
     },
     630,
-    )
+);
-});
 
 #[track_caller]
 fn run_single_threaded(bencher: Bencher, benchmark: &Benchmark) {
@@ -245,22 +238,22 @@ fn run_single_threaded(bencher: Bencher, benchmark: &Benchmark) {
         });
 }
 
-#[bench(args=[&*ALTAIR, &*FREQTRADE, &*PYDANTIC, &*TANJUN], sample_size=2, sample_count=3)]
+#[bench(args=[&ALTAIR, &FREQTRADE, &PYDANTIC, &TANJUN], sample_size=2, sample_count=3)]
 fn small(bencher: Bencher, benchmark: &Benchmark) {
     run_single_threaded(bencher, benchmark);
 }
 
-#[bench(args=[&*COLOUR_SCIENCE, &*PANDAS, &*STATIC_FRAME], sample_size=1, sample_count=3)]
+#[bench(args=[&COLOUR_SCIENCE, &PANDAS, &STATIC_FRAME], sample_size=1, sample_count=3)]
 fn medium(bencher: Bencher, benchmark: &Benchmark) {
     run_single_threaded(bencher, benchmark);
 }
 
-#[bench(args=[&*SYMPY], sample_size=1, sample_count=2)]
+#[bench(args=[&SYMPY], sample_size=1, sample_count=2)]
 fn large(bencher: Bencher, benchmark: &Benchmark) {
     run_single_threaded(bencher, benchmark);
 }
 
-#[bench(args=[&*PYDANTIC], sample_size=3, sample_count=8)]
+#[bench(args=[&PYDANTIC], sample_size=3, sample_count=8)]
 fn multithreaded(bencher: Bencher, benchmark: &Benchmark) {
     let thread_pool = ThreadPoolBuilder::new().build().unwrap();
 

crates/ruff_benchmark/src/real_world_projects.rs

@@ -30,9 +30,9 @@ pub struct RealWorldProject<'a> {
     /// Specific commit hash to checkout
     pub commit: &'a str,
     /// List of paths within the project to check (`ty check <paths>`)
-    pub paths: Vec<&'a SystemPath>,
+    pub paths: &'a [&'a str],
     /// Dependencies to install via uv
-    pub dependencies: Vec<&'a str>,
+    pub dependencies: &'a [&'a str],
     /// Limit candidate packages to those that were uploaded prior to a given point in time (ISO 8601 format).
     /// Maps to uv's `exclude-newer`.
     pub max_dep_date: &'a str,
@@ -125,9 +125,9 @@ impl<'a> InstalledProject<'a> {
         &self.config
     }
 
-    /// Get the benchmark paths as `SystemPathBuf`
+    /// Get the benchmark paths
-    pub fn check_paths(&self) -> &[&SystemPath] {
+    pub fn check_paths(&self) -> &[&str] {
-        &self.config.paths
+        self.config.paths
     }
 
     /// Get the virtual environment path
@@ -297,7 +297,7 @@ fn install_dependencies(checkout: &Checkout) -> Result<()> {
         "--exclude-newer",
         checkout.project().max_dep_date,
     ])
-    .args(&checkout.project().dependencies);
+    .args(checkout.project().dependencies);
 
     let output = cmd
         .output()