Build codspeed benchmarks by calling cargo directly (#10735)

.github/workflows/ci.yaml

@@ -525,8 +525,23 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
 
+      # Codspeed comes with a very ancient cargo version (1.66) that resolves features flags differently than what we use now.
+      # This can result in build failures; see https://github.com/astral-sh/ruff/pull/10700.
+      # There's a pending codspeed PR to upgrade to a newer cargo version, but until that's merged, we need to use the workaround below.
+      # https://github.com/CodSpeedHQ/codspeed-rust/pull/31
+      # What we do is to call cargo build manually with the correct feature flags and RUSTC settings. We'll have to
+      # manually maintain the list of benchmarks to run with codspeed (the benefit is that we could detect which benchmarks to run and build based on the changes).
+      # This is inspired by https://github.com/oxc-project/oxc/blob/a0532adc654039a0c7ead7b35216dfa0b0cb8e8f/.github/workflows/benchmark.yml
       - name: "Build benchmarks"
-        run: cargo codspeed build --features codspeed -p ruff_benchmark
+        env:
+          RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
+        shell: bash
+        # Build all benchmarks, copy the binary to the codspeed directory, remove any `*.d` files that might have been created.
+        run: |
+          cargo build --release -p ruff_benchmark --bench parser --bench linter --bench formatter --bench lexer --features=codspeed
+          mkdir -p ./target/codspeed/ruff_benchmark
+          cp ./target/release/deps/{lexer,parser,linter,formatter}* target/codspeed/ruff_benchmark/
+          rm -rf ./target/codspeed/ruff_benchmark/*.d
 
       - name: "Run benchmarks"
         uses: CodSpeedHQ/action@v2