From ea6730f5463587f48cddc84bb3f8c31cf6a6a5b1 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Mon, 22 Dec 2025 15:06:22 +0100 Subject: [PATCH] [ty] Speed-up instrumented benchmarks (#22133) --- .github/actionlint.yaml | 1 + .github/workflows/ci.yaml | 60 ++++++++++++++++++++++++++------ crates/ruff_benchmark/Cargo.toml | 37 ++++++++++++-------- crates/ruff_benchmark/src/lib.rs | 2 +- 4 files changed, 73 insertions(+), 27 deletions(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 5813495fb6..5e0664dfc7 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -4,6 +4,7 @@ self-hosted-runner: # Various runners we use that aren't recognized out-of-the-box by actionlint: labels: + - depot-ubuntu-24.04-4 - depot-ubuntu-latest-8 - depot-ubuntu-22.04-16 - depot-ubuntu-22.04-32 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 02eebb6e6a..967389d1fe 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -952,7 +952,7 @@ jobs: tool: cargo-codspeed - name: "Build benchmarks" - run: cargo codspeed build --features "codspeed,instrumented" --profile profiling --no-default-features -p ruff_benchmark --bench formatter --bench lexer --bench linter --bench parser + run: cargo codspeed build --features "codspeed,ruff_instrumented" --profile profiling --no-default-features -p ruff_benchmark --bench formatter --bench lexer --bench linter --bench parser - name: "Run benchmarks" uses: CodSpeedHQ/action@346a2d8a8d9d38909abd0bc3d23f773110f076ad # v4.4.1 @@ -960,9 +960,9 @@ jobs: mode: simulation run: cargo codspeed run - benchmarks-instrumented-ty: - name: "benchmarks instrumented (ty)" - runs-on: ubuntu-24.04 + benchmarks-instrumented-ty-build: + name: "benchmarks instrumented ty (build)" + runs-on: depot-ubuntu-24.04-4 needs: determine_changes if: | github.repository == 'astral-sh/ruff' && @@ -971,9 +971,6 @@ jobs: needs.determine_changes.outputs.ty == 'true' ) timeout-minutes: 20 - permissions: - contents: read # required for actions/checkout - id-token: write # required for OIDC authentication with CodSpeed steps: - name: "Checkout Branch" uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -983,7 +980,6 @@ jobs: - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 with: save-if: ${{ github.ref == 'refs/heads/main' }} - - uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4 - name: "Install Rust toolchain" run: rustup show @@ -994,13 +990,55 @@ jobs: tool: cargo-codspeed - name: "Build benchmarks" - run: cargo codspeed build --features "codspeed,instrumented" --profile profiling --no-default-features -p ruff_benchmark --bench ty + run: cargo codspeed build -m instrumentation --features "codspeed,ty_instrumented" --profile profiling --no-default-features -p ruff_benchmark --bench ty + + - name: "Upload benchmark binary" + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: benchmarks-instrumented-ty-binary + path: target/codspeed/instrumentation/ruff_benchmark + retention-days: 1 + + benchmarks-instrumented-ty-run: + name: "benchmarks instrumented ty (${{ matrix.benchmark }})" + runs-on: ubuntu-24.04 + needs: benchmarks-instrumented-ty-build + timeout-minutes: 20 + permissions: + contents: read # required for actions/checkout + id-token: write # required for OIDC authentication with CodSpeed + strategy: + fail-fast: false + matrix: + benchmark: + - "check_file|micro|anyio" + - "attrs|hydra|datetype" + steps: + - name: "Checkout Branch" + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4 + + - name: "Install codspeed" + uses: taiki-e/install-action@3575e532701a5fc614b0c842e4119af4cc5fd16d # v2.62.60 + with: + tool: cargo-codspeed + + - name: "Download benchmark binary" + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v4.3.0 + with: + name: benchmarks-instrumented-ty-binary + path: target/codspeed/instrumentation/ruff_benchmark + + - name: "Restore binary permissions" + run: chmod +x target/codspeed/instrumentation/ruff_benchmark/ty - name: "Run benchmarks" uses: CodSpeedHQ/action@346a2d8a8d9d38909abd0bc3d23f773110f076ad # v4.4.1 with: mode: simulation - run: cargo codspeed run + run: cargo codspeed run --bench ty "${{ matrix.benchmark }}" benchmarks-walltime-build: name: "benchmarks walltime (build)" @@ -1030,7 +1068,7 @@ jobs: tool: cargo-codspeed - name: "Build benchmarks" - run: cargo codspeed build -m walltime --features "codspeed,walltime" --profile profiling --no-default-features -p ruff_benchmark + run: cargo codspeed build -m walltime --features "codspeed,ty_walltime" --profile profiling --no-default-features -p ruff_benchmark - name: "Upload benchmark binary" uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 diff --git a/crates/ruff_benchmark/Cargo.toml b/crates/ruff_benchmark/Cargo.toml index d10c45f452..2a86fff739 100644 --- a/crates/ruff_benchmark/Cargo.toml +++ b/crates/ruff_benchmark/Cargo.toml @@ -19,32 +19,32 @@ doctest = false [[bench]] name = "linter" harness = false -required-features = ["instrumented"] +required-features = ["ruff_instrumented"] [[bench]] name = "lexer" harness = false -required-features = ["instrumented"] +required-features = ["ruff_instrumented"] [[bench]] name = "parser" harness = false -required-features = ["instrumented"] +required-features = ["ruff_instrumented"] [[bench]] name = "formatter" harness = false -required-features = ["instrumented"] +required-features = ["ruff_instrumented"] [[bench]] name = "ty" harness = false -required-features = ["instrumented"] +required-features = ["ty_instrumented"] [[bench]] name = "ty_walltime" harness = false -required-features = ["walltime"] +required-features = ["ty_walltime"] [dependencies] ruff_db = { workspace = true, features = ["testing"] } @@ -67,25 +67,32 @@ tracing = { workspace = true } workspace = true [features] -default = ["instrumented", "walltime"] -# Enables the benchmark that should only run with codspeed's instrumented runner -instrumented = [ +default = ["ty_instrumented", "ty_walltime", "ruff_instrumented"] +# Enables the ruff instrumented benchmarks +ruff_instrumented = [ "criterion", "ruff_linter", "ruff_python_formatter", "ruff_python_parser", "ruff_python_trivia", + "mimalloc", + "tikv-jemallocator", +] +# Enables the ty instrumented benchmarks +ty_instrumented = [ + "criterion", "ty_project", + "ruff_python_trivia", ] codspeed = ["codspeed-criterion-compat"] -# Enables benchmark that should only run with codspeed's walltime runner. -walltime = ["ruff_db/os", "ty_project", "divan"] +# Enables the ty_walltime benchmarks +ty_walltime = ["ruff_db/os", "ty_project", "divan"] -[target.'cfg(target_os = "windows")'.dev-dependencies] -mimalloc = { workspace = true } +[target.'cfg(target_os = "windows")'.dependencies] +mimalloc = { workspace = true, optional = true } -[target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64", target_arch = "riscv64")))'.dev-dependencies] -tikv-jemallocator = { workspace = true } +[target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64", target_arch = "riscv64")))'.dependencies] +tikv-jemallocator = { workspace = true, optional = true } [dev-dependencies] rustc-hash = { workspace = true } diff --git a/crates/ruff_benchmark/src/lib.rs b/crates/ruff_benchmark/src/lib.rs index 34ba0d6364..a3b9514fbd 100644 --- a/crates/ruff_benchmark/src/lib.rs +++ b/crates/ruff_benchmark/src/lib.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -#[cfg(feature = "instrumented")] +#[cfg(any(feature = "ty_instrumented", feature = "ruff_instrumented"))] pub mod criterion; pub mod real_world_projects;