[ty] Speed-up instrumented benchmarks (#22133)

.github/actionlint.yaml

@@ -4,6 +4,7 @@
 self-hosted-runner:
   # Various runners we use that aren't recognized out-of-the-box by actionlint:
   labels:
+    - depot-ubuntu-24.04-4
     - depot-ubuntu-latest-8
     - depot-ubuntu-22.04-16
     - depot-ubuntu-22.04-32

.github/workflows/ci.yaml

@@ -952,7 +952,7 @@ jobs:
           tool: cargo-codspeed
 
       - name: "Build benchmarks"
-        run: cargo codspeed build --features "codspeed,instrumented" --profile profiling --no-default-features -p ruff_benchmark --bench formatter --bench lexer --bench linter --bench parser
+        run: cargo codspeed build --features "codspeed,ruff_instrumented" --profile profiling --no-default-features -p ruff_benchmark --bench formatter --bench lexer --bench linter --bench parser
 
       - name: "Run benchmarks"
         uses: CodSpeedHQ/action@346a2d8a8d9d38909abd0bc3d23f773110f076ad # v4.4.1
@@ -960,9 +960,9 @@ jobs:
           mode: simulation
           run: cargo codspeed run
 
-  benchmarks-instrumented-ty:
-    name: "benchmarks instrumented (ty)"
-    runs-on: ubuntu-24.04
+  benchmarks-instrumented-ty-build:
+    name: "benchmarks instrumented ty (build)"
+    runs-on: depot-ubuntu-24.04-4
     needs: determine_changes
     if: |
       github.repository == 'astral-sh/ruff' &&
@@ -971,9 +971,6 @@ jobs:
         needs.determine_changes.outputs.ty == 'true'
       )
     timeout-minutes: 20
-    permissions:
-      contents: read # required for actions/checkout
-      id-token: write # required for OIDC authentication with CodSpeed
     steps:
       - name: "Checkout Branch"
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -983,7 +980,6 @@ jobs:
       - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2
         with:
           save-if: ${{ github.ref == 'refs/heads/main' }}
-      - uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4
 
       - name: "Install Rust toolchain"
         run: rustup show
@@ -994,13 +990,55 @@ jobs:
           tool: cargo-codspeed
 
       - name: "Build benchmarks"
-        run: cargo codspeed build --features "codspeed,instrumented" --profile profiling --no-default-features -p ruff_benchmark --bench ty
+        run: cargo codspeed build -m instrumentation --features "codspeed,ty_instrumented" --profile profiling --no-default-features -p ruff_benchmark --bench ty
+
+      - name: "Upload benchmark binary"
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: benchmarks-instrumented-ty-binary
+          path: target/codspeed/instrumentation/ruff_benchmark
+          retention-days: 1
+
+  benchmarks-instrumented-ty-run:
+    name: "benchmarks instrumented ty (${{ matrix.benchmark }})"
+    runs-on: ubuntu-24.04
+    needs: benchmarks-instrumented-ty-build
+    timeout-minutes: 20
+    permissions:
+      contents: read # required for actions/checkout
+      id-token: write # required for OIDC authentication with CodSpeed
+    strategy:
+      fail-fast: false
+      matrix:
+        benchmark:
+          - "check_file|micro|anyio"
+          - "attrs|hydra|datetype"
+    steps:
+      - name: "Checkout Branch"
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4
+
+      - name: "Install codspeed"
+        uses: taiki-e/install-action@3575e532701a5fc614b0c842e4119af4cc5fd16d # v2.62.60
+        with:
+          tool: cargo-codspeed
+
+      - name: "Download benchmark binary"
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v4.3.0
+        with:
+          name: benchmarks-instrumented-ty-binary
+          path: target/codspeed/instrumentation/ruff_benchmark
+
+      - name: "Restore binary permissions"
+        run: chmod +x target/codspeed/instrumentation/ruff_benchmark/ty
 
       - name: "Run benchmarks"
         uses: CodSpeedHQ/action@346a2d8a8d9d38909abd0bc3d23f773110f076ad # v4.4.1
         with:
           mode: simulation
-          run: cargo codspeed run
+          run: cargo codspeed run --bench ty "${{ matrix.benchmark }}"
 
   benchmarks-walltime-build:
     name: "benchmarks walltime (build)"
@@ -1030,7 +1068,7 @@ jobs:
           tool: cargo-codspeed
 
       - name: "Build benchmarks"
-        run: cargo codspeed build -m walltime --features "codspeed,walltime" --profile profiling --no-default-features -p ruff_benchmark
+        run: cargo codspeed build -m walltime --features "codspeed,ty_walltime" --profile profiling --no-default-features -p ruff_benchmark
 
       - name: "Upload benchmark binary"
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2

crates/ruff_benchmark/Cargo.toml

@@ -19,32 +19,32 @@ doctest = false
 [[bench]]
 name = "linter"
 harness = false
-required-features = ["instrumented"]
+required-features = ["ruff_instrumented"]
 
 [[bench]]
 name = "lexer"
 harness = false
-required-features = ["instrumented"]
+required-features = ["ruff_instrumented"]
 
 [[bench]]
 name = "parser"
 harness = false
-required-features = ["instrumented"]
+required-features = ["ruff_instrumented"]
 
 [[bench]]
 name = "formatter"
 harness = false
-required-features = ["instrumented"]
+required-features = ["ruff_instrumented"]
 
 [[bench]]
 name = "ty"
 harness = false
-required-features = ["instrumented"]
+required-features = ["ty_instrumented"]
 
 [[bench]]
 name = "ty_walltime"
 harness = false
-required-features = ["walltime"]
+required-features = ["ty_walltime"]
 
 [dependencies]
 ruff_db = { workspace = true, features = ["testing"] }
@@ -67,25 +67,32 @@ tracing = { workspace = true }
 workspace = true
 
 [features]
-default = ["instrumented", "walltime"]
-# Enables the benchmark that should only run with codspeed's instrumented runner
-instrumented = [
+default = ["ty_instrumented", "ty_walltime", "ruff_instrumented"]
+# Enables the ruff instrumented benchmarks
+ruff_instrumented = [
     "criterion",
     "ruff_linter",
     "ruff_python_formatter",
     "ruff_python_parser",
     "ruff_python_trivia",
+    "mimalloc",
+    "tikv-jemallocator",
+]
+# Enables the ty instrumented benchmarks
+ty_instrumented = [
+    "criterion",
     "ty_project",
+    "ruff_python_trivia",
 ]
 codspeed = ["codspeed-criterion-compat"]
-# Enables benchmark that should only run with codspeed's walltime runner.
-walltime = ["ruff_db/os", "ty_project", "divan"]
+# Enables the ty_walltime benchmarks
+ty_walltime = ["ruff_db/os", "ty_project", "divan"]
 
-[target.'cfg(target_os = "windows")'.dev-dependencies]
-mimalloc = { workspace = true }
+[target.'cfg(target_os = "windows")'.dependencies]
+mimalloc = { workspace = true, optional = true }
 
-[target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64", target_arch = "riscv64")))'.dev-dependencies]
-tikv-jemallocator = { workspace = true }
+[target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64", target_arch = "riscv64")))'.dependencies]
+tikv-jemallocator = { workspace = true, optional = true }
 
 [dev-dependencies]
 rustc-hash = { workspace = true }

crates/ruff_benchmark/src/lib.rs

@@ -1,6 +1,6 @@
 use std::path::PathBuf;
 
-#[cfg(feature = "instrumented")]
+#[cfg(any(feature = "ty_instrumented", feature = "ruff_instrumented"))]
 pub mod criterion;
 pub mod real_world_projects;