[ty] Speedup ty-walltime benchmarks (#22126)

Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>

.github/actionlint.yaml

@@ -8,6 +8,7 @@ self-hosted-runner:
     - depot-ubuntu-22.04-16
     - depot-ubuntu-22.04-32
     - depot-windows-2022-16
+    - depot-ubuntu-22.04-arm-4
     - github-windows-2025-x86_64-8
     - github-windows-2025-x86_64-16
     - codspeed-macro

.github/workflows/ci.yaml

@@ -1002,20 +1002,14 @@ jobs:
           mode: simulation
           run: cargo codspeed run
 
-  benchmarks-walltime:
-    name: "benchmarks walltime (${{ matrix.benchmarks }})"
-    runs-on: codspeed-macro
+  benchmarks-walltime-build:
+    name: "benchmarks walltime (build)"
+    # We only run this job if `github.repository == 'astral-sh/ruff'`,
+    # so hardcoding depot here is fine
+    runs-on: depot-ubuntu-22.04-arm-4
     needs: determine_changes
     if: ${{ github.repository == 'astral-sh/ruff' && !contains(github.event.pull_request.labels.*.name, 'no-test') && (needs.determine_changes.outputs.ty == 'true' || github.ref == 'refs/heads/main') }}
     timeout-minutes: 20
-    permissions:
-      contents: read # required for actions/checkout
-      id-token: write # required for OIDC authentication with CodSpeed
-    strategy:
-      matrix:
-        benchmarks:
-          - "medium|multithreaded"
-          - "small|large"
     steps:
       - name: "Checkout Branch"
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -1036,7 +1030,51 @@ jobs:
           tool: cargo-codspeed
 
       - name: "Build benchmarks"
-        run: cargo codspeed build --features "codspeed,walltime" --profile profiling --no-default-features -p ruff_benchmark
+        run: cargo codspeed build -m walltime --features "codspeed,walltime" --profile profiling --no-default-features -p ruff_benchmark
+
+      - name: "Upload benchmark binary"
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: benchmarks-walltime-binary
+          path: target/codspeed/walltime/ruff_benchmark
+          retention-days: 1
+
+  benchmarks-walltime-run:
+    name: "benchmarks walltime (${{ matrix.benchmark }})"
+    runs-on: codspeed-macro
+    needs: benchmarks-walltime-build
+    timeout-minutes: 20
+    permissions:
+      contents: read # required for actions/checkout
+      id-token: write # required for OIDC authentication with CodSpeed
+    strategy:
+      matrix:
+        benchmark:
+          - colour_science
+          - "pandas|tanjun|altair"
+          - "static_frame|sympy"
+          - "pydantic|multithreaded|freqtrade"
+    steps:
+      - name: "Checkout Branch"
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          persist-credentials: false
+
+      - uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4
+
+      - name: "Install codspeed"
+        uses: taiki-e/install-action@3575e532701a5fc614b0c842e4119af4cc5fd16d # v2.62.60
+        with:
+          tool: cargo-codspeed
+
+      - name: "Download benchmark binary"
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        with:
+          name: benchmarks-walltime-binary
+          path: target/codspeed/walltime/ruff_benchmark
+
+      - name: "Restore binary permissions"
+        run: chmod +x target/codspeed/walltime/ruff_benchmark/ty_walltime
 
       - name: "Run benchmarks"
         uses: CodSpeedHQ/action@346a2d8a8d9d38909abd0bc3d23f773110f076ad # v4.4.1
@@ -1047,4 +1085,4 @@ jobs:
           CODSPEED_PERF_ENABLED: false
         with:
           mode: walltime
-          run: cargo codspeed run --bench ty_walltime "${{ matrix.benchmarks }}"
+          run: cargo codspeed run --bench ty_walltime -m walltime "${{ matrix.benchmark }}"

crates/ruff_benchmark/benches/ty_walltime.rs

@@ -235,30 +235,55 @@ fn run_single_threaded(bencher: Bencher, benchmark: &Benchmark) {
         });
 }
 
-#[bench(args=[&ALTAIR, &FREQTRADE, &TANJUN], sample_size=2, sample_count=3)]
-fn small(bencher: Bencher, benchmark: &Benchmark) {
-    run_single_threaded(bencher, benchmark);
+#[bench(sample_size = 2, sample_count = 3)]
+fn altair(bencher: Bencher) {
+    run_single_threaded(bencher, &ALTAIR);
 }
 
-#[bench(args=[&COLOUR_SCIENCE, &PANDAS, &STATIC_FRAME], sample_size=1, sample_count=3)]
-fn medium(bencher: Bencher, benchmark: &Benchmark) {
-    run_single_threaded(bencher, benchmark);
+#[bench(sample_size = 2, sample_count = 3)]
+fn freqtrade(bencher: Bencher) {
+    run_single_threaded(bencher, &FREQTRADE);
 }
 
-#[bench(args=[&SYMPY, &PYDANTIC], sample_size=1, sample_count=2)]
-fn large(bencher: Bencher, benchmark: &Benchmark) {
-    run_single_threaded(bencher, benchmark);
+#[bench(sample_size = 2, sample_count = 3)]
+fn tanjun(bencher: Bencher) {
+    run_single_threaded(bencher, &TANJUN);
 }
 
-#[bench(args=[&ALTAIR], sample_size=3, sample_count=8)]
-fn multithreaded(bencher: Bencher, benchmark: &Benchmark) {
+#[bench(sample_size = 2, sample_count = 3)]
+fn pydantic(bencher: Bencher) {
+    run_single_threaded(bencher, &PYDANTIC);
+}
+
+#[bench(sample_size = 1, sample_count = 3)]
+fn static_frame(bencher: Bencher) {
+    run_single_threaded(bencher, &STATIC_FRAME);
+}
+
+#[bench(sample_size = 1, sample_count = 2)]
+fn colour_science(bencher: Bencher) {
+    run_single_threaded(bencher, &COLOUR_SCIENCE);
+}
+
+#[bench(sample_size = 1, sample_count = 2)]
+fn pandas(bencher: Bencher) {
+    run_single_threaded(bencher, &PANDAS);
+}
+
+#[bench(sample_size = 1, sample_count = 2)]
+fn sympy(bencher: Bencher) {
+    run_single_threaded(bencher, &SYMPY);
+}
+
+#[bench(sample_size = 3, sample_count = 8)]
+fn multithreaded(bencher: Bencher) {
     let thread_pool = ThreadPoolBuilder::new().build().unwrap();
 
     bencher
-        .with_inputs(|| benchmark.setup_iteration())
+        .with_inputs(|| ALTAIR.setup_iteration())
         .bench_local_values(|db| {
             thread_pool.install(|| {
-                check_project(&db, benchmark.project.name, benchmark.max_diagnostics);
+                check_project(&db, ALTAIR.project.name, ALTAIR.max_diagnostics);
                 db
             })
         });