Use Codspeed for continous benchmarking (#6896)

.github/workflows/benchmark.yaml

@@ -1,136 +0,0 @@
-name: Benchmark
-
-on:
-  pull_request:
-    paths:
-      - "Cargo.toml"
-      - "Cargo.lock"
-      - "rust-toolchain"
-      - "crates/**"
-      - "!crates/ruff_dev"
-      - "!crates/ruff_shrinking"
-
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: true
-
-jobs:
-  run-benchmark:
-    if: github.event_name == 'pull_request'
-    name: "Run | ${{ matrix.os }}"
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - name: "PR - Checkout Branch"
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: "PR - Install Rust toolchain"
-        run: rustup show
-
-      - uses: Swatinem/rust-cache@v2
-
-      - name: "PR - Build benchmarks"
-        run: cargo bench -p ruff_benchmark --no-run
-
-      - name: "PR - Run benchmarks"
-        run: cargo benchmark --save-baseline=pr
-
-      - name: "Main - Checkout Branch"
-        uses: actions/checkout@v3
-        with:
-          clean: false
-          ref: main
-
-      - name: "Main - Install Rust toolchain"
-        run: rustup show
-
-      - name: "Main - Build benchmarks"
-        run: cargo bench -p ruff_benchmark --no-run
-
-      - name: "Main - Run benchmarks"
-        run: cargo benchmark --save-baseline=main
-
-      - name: "Upload benchmark results"
-        uses: actions/upload-artifact@v3
-        with:
-          name: benchmark-results-${{ matrix.os }}
-          path: ./target/criterion
-
-      # Cleanup
-      - name: Remove Criterion Artifact
-        uses: JesseTG/rm@v1.0.3
-        with:
-          path: ./target/criterion
-
-  benchmark-compare:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    name: Compare
-    needs:
-      - run-benchmark
-
-    steps:
-      - name: "Install Rust toolchain"
-        run: rustup show
-
-      - name: "Install critcmp"
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-
-      - name: "Linux | Download PR benchmark results"
-        uses: actions/download-artifact@v3
-        with:
-          name: benchmark-results-ubuntu-latest
-          path: ./target/criterion
-
-      - name: "Linux | Compare benchmark results"
-        shell: bash
-        run: |
-          echo "### Benchmark" >> summary.md
-          echo "#### Linux" >> summary.md
-          echo "\`\`\`" >> summary.md
-          critcmp main pr >> summary.md
-          echo "\`\`\`" >> summary.md
-          echo "" >> summary.md
-
-      - name: "Linux | Cleanup benchmark results"
-        run: rm -rf ./target/criterion
-
-      - name: "Windows | Download PR benchmark results"
-        uses: actions/download-artifact@v3
-        with:
-          name: benchmark-results-windows-latest
-          path: ./target/criterion
-
-      - name: "Windows | Compare benchmark results"
-        shell: bash
-        run: |
-          echo "#### Windows" >> summary.md
-          echo "\`\`\`" >> summary.md
-          critcmp main pr >> summary.md
-          echo "\`\`\`" >> summary.md
-          echo "" >> summary.md
-
-          echo ${{ github.event.pull_request.number }} > pr-number
-
-          cat summary.md > $GITHUB_STEP_SUMMARY
-
-      - uses: actions/upload-artifact@v3
-        name: Upload PR Number
-        with:
-          name: pr-number
-          path: pr-number
-
-      - uses: actions/upload-artifact@v3
-        name: Upload Summary
-        with:
-          name: summary
-          path: summary.md

.github/workflows/ci.yaml

@@ -341,3 +341,28 @@ jobs:
         run: cat target/progress_projects_stats.txt > $GITHUB_STEP_SUMMARY
       - name: "Remove checkouts from cache"
         run: rm -r target/progress_projects
+
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout Branch"
+        uses: actions/checkout@v3
+
+      - name: "Install Rust toolchain"
+        run: rustup show
+
+      - name: "Install codspeed"
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-codspeed
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: "Build benchmarks"
+        run: cargo codspeed build --features codspeed -p ruff_benchmark
+
+      - name: "Run benchmarks"
+        uses: CodSpeedHQ/action@v1
+        with:
+          run: cargo codspeed run
+          token: ${{ secrets.CODSPEED_TOKEN }}

.github/workflows/pr-comment.yaml

@@ -2,7 +2,7 @@ name: PR Check Comment
 
 on:
   workflow_run:
-    workflows: [CI, Benchmark]
+    workflows: [CI]
     types: [completed]
   workflow_dispatch:
     inputs:
@@ -43,35 +43,16 @@ jobs:
           path: pr/ecosystem
           if_no_artifact_found: ignore
 
-      - uses: dawidd6/action-download-artifact@v2
-        name: "Download Benchmark Result"
-        id: download-benchmark-result
-        if: steps.pr-number.outputs.pr-number
-        with:
-          name: summary
-          workflow: benchmark.yaml
-          pr: ${{ steps.pr-number.outputs.pr-number }}
-          path: pr/benchmark
-          if_no_artifact_found: ignore
-
       - name: Generate Comment
         id: generate-comment
-        if: steps.download-ecosystem-result.outputs.found_artifact == 'true' || steps.download-benchmark-result.outputs.found_artifact == 'true'
+        if: steps.download-ecosystem-result.outputs.found_artifact == 'true'
         run: |
           echo 'comment<<EOF' >> $GITHUB_OUTPUT
           echo '## PR Check Results' >> $GITHUB_OUTPUT
 
-          if [[ -f pr/ecosystem/ecosystem-result ]]
-          then
-            echo "### Ecosystem" >> $GITHUB_OUTPUT
-            cat pr/ecosystem/ecosystem-result >> $GITHUB_OUTPUT
-            echo "" >> $GITHUB_OUTPUT
-          fi
-
-          if [[ -f pr/benchmark/summary.md ]]
-          then
-            cat pr/benchmark/summary.md >> $GITHUB_OUTPUT
-          fi
+          echo "### Ecosystem" >> $GITHUB_OUTPUT
+          cat pr/ecosystem/ecosystem-result >> $GITHUB_OUTPUT
+          echo "" >> $GITHUB_OUTPUT
 
           echo 'EOF' >> $GITHUB_OUTPUT