ci: Benchmark CI Step (#3480)

.github/workflows/benchmark.yaml

@@ -0,0 +1,133 @@
+name: Benchmark
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  run-benchmark:
+    if: github.event_name == 'pull_request'
+    name: "Run | ${{ matrix.os }}"
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: "PR - Checkout Branch"
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: "PR - Install Rust toolchain"
+        run: rustup show
+
+      - uses: Swatinem/rust-cache@v1
+
+      - name: "PR - Build benchmarks"
+        uses: actions-rs/cargo@v1
+        with:
+          command: bench
+          args: -p ruff_benchmark --no-run
+
+      - name: "PR - Run benchmarks"
+        run: cargo benchmark --save-baseline=pr
+
+      - name: "Main - Checkout Branch"
+        uses: actions/checkout@v3
+        with:
+          clean: false
+          ref: main
+
+      - name: "Main - Install Rust toolchain"
+        run: rustup show
+
+      - name: "Main - Build benchmarks"
+        uses: actions-rs/cargo@v1
+        with:
+          command: bench
+          args: -p ruff_benchmark --no-run
+
+      - name: "Main - Run benchmarks"
+        run: cargo benchmark --save-baseline=main
+
+      - name: "Upload benchmark results"
+        uses: actions/upload-artifact@v3
+        with:
+          name: benchmark-results-${{ matrix.os }}
+          path: ./target/criterion
+
+      # Cleanup
+      - name: Remove Criterion Artifact
+        uses: JesseTG/rm@v1.0.3
+        with:
+          path: ./target/criterion
+
+  benchmark-compare:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    name: Compare
+    needs:
+      - run-benchmark
+
+    steps:
+    - name: "Install Rust toolchain"
+      run: rustup show
+
+    - name: "Install critcmp"
+      # Use debug build: Building takes much longer than the "slowness" of using the debug build.
+      run: cargo install --debug critcmp
+
+    - name: "Linux | Download PR benchmark results"
+      uses: actions/download-artifact@v3
+      with:
+        name: benchmark-results-ubuntu-latest
+        path: ./target/criterion
+
+    - name: "Linux | Compare benchmark results"
+      shell: bash
+      run: |
+        echo "### Benchmark" >> summary.md
+        echo "#### Linux" >> summary.md
+        echo "\`\`\`" >> summary.md
+        critcmp main pr >> summary.md
+        echo "\`\`\`" >> summary.md
+        echo "" >> summary.md
+
+    - name: "Linux | Cleanup benchmark results"
+      run: rm -rf ./target/criterion
+
+    - name: "Windows | Download PR benchmark results"
+      uses: actions/download-artifact@v3
+      with:
+        name: benchmark-results-windows-latest
+        path: ./target/criterion
+
+    - name: "Windows | Compare benchmark results"
+      shell: bash
+      run: |
+        echo "#### Windows" >> summary.md
+        echo "\`\`\`" >> summary.md
+        critcmp main pr >> summary.md
+        echo "\`\`\`" >> summary.md
+        echo "" >> summary.md
+
+        echo ${{ github.event.pull_request.number }} > pr-number
+
+        cat summary.md > $GITHUB_STEP_SUMMARY
+
+    - uses: actions/upload-artifact@v3
+      name: Upload PR Number
+      with:
+        name: pr-number
+        path: pr-number
+
+    - uses: actions/upload-artifact@v3
+      name: Upload Summary
+      with:
+        name: summary
+        path: summary.md

.github/workflows/ci.yaml

@@ -80,6 +80,7 @@ jobs:
           # Setting RUSTDOCFLAGS because `cargo doc --check` isn't yet implemented (https://github.com/rust-lang/cargo/issues/10025).
           RUSTDOCFLAGS: "-D warnings"
       - uses: actions/upload-artifact@v3
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: ruff
           path: target/debug/ruff
@@ -139,27 +140,39 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.11"
+
       - uses: actions/download-artifact@v3
+        name: Download Ruff binary
         id: ruff-target
         with:
           name: ruff
           path: target/debug
+
       - uses: dawidd6/action-download-artifact@v2
+        name: Download base results
         with:
           name: ruff
           branch: ${{ github.event.pull_request.base.ref }}
           check_artifacts: true
+
       - name: Run ecosystem check
         run: |
           # Make executable, since artifact download doesn't preserve this
           chmod +x ruff ${{ steps.ruff-target.outputs.download-path }}/ruff
 
           scripts/check_ecosystem.py ruff ${{ steps.ruff-target.outputs.download-path }}/ruff | tee ecosystem-result
+          cat ecosystem-result > $GITHUB_STEP_SUMMARY
 
           echo ${{ github.event.number }} > pr-number
+
       - uses: actions/upload-artifact@v3
+        name: Upload PR Number
+        with:
+          name: pr-number
+          path: pr-number
+
+      - uses: actions/upload-artifact@v3
+        name: Upload Results
         with:
           name: ecosystem-result
-          path: |
+          path: ecosystem-result
-            ecosystem-result
-            pr-number

.github/workflows/pr-comment.yaml

@@ -2,7 +2,7 @@ name: PR Check Comment
 
 on:
   workflow_run:
-    workflows: [CI]
+    workflows: [CI, Benchmark]
     types: [completed]
   workflow_dispatch:
     inputs:
@@ -17,22 +17,67 @@ jobs:
   comment:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
       - uses: dawidd6/action-download-artifact@v2
-        id: download-result
+        name: Download PR Number
+        with:
+          name: pr-number
+          run_id: ${{ github.event.workflow_run.id ||  github.event.inputs.workflow_run_id }}
+          if_no_artifact_found: ignore
+
+      - name: Extract PR Number
+        id: pr-number
+        run: |
+          if [[ -f pr-number ]]
+          then
+            echo "pr-number=$(<pr-number)" >> $GITHUB_OUTPUT
+          fi
+
+      - uses: dawidd6/action-download-artifact@v2
+        name: "Download Ecosystem Result"
+        id: download-ecosystem-result
+        if: steps.pr-number.outputs.pr-number
         with:
           name: ecosystem-result
           workflow: ci.yaml
-          run_id: ${{ github.event.workflow_run.id || github.event.inputs.workflow_run_id }}
+          pr: ${{ steps.pr-number.outputs.pr-number }}
+          path: pr/ecosystem
           if_no_artifact_found: ignore
-      - if: steps.download-result.outputs.found_artifact
+
-        id: result
+      - uses: dawidd6/action-download-artifact@v2
+        name: "Download Benchmark Result"
+        id: download-benchmark-result
+        if: steps.pr-number.outputs.pr-number
+        with:
+          name: summary
+          workflow: benchmark.yaml
+          pr: ${{ steps.pr-number.outputs.pr-number }}
+          path: pr/benchmark
+          if_no_artifact_found: ignore
+
+      - name: Generate Comment
+        id: generate-comment
+        if: steps.download-ecosystem-result.outputs.found_artifact == 'true' || steps.download-benchmark-result.outputs.found_artifact == 'true'
         run: |
-          echo "pr-number=$(<pr-number)" >> $GITHUB_OUTPUT
+          echo 'comment<<EOF' >> $GITHUB_OUTPUT
-      - name: Create comment
+          echo '## PR Check Results' >> $GITHUB_OUTPUT
-        if: steps.download-result.outputs.found_artifact
+
+          if [[ -f pr/ecosystem/ecosystem-result ]]
+          then
+            echo "### Ecosystem" >> $GITHUB_OUTPUT
+            cat pr/ecosystem/ecosystem-result >> $GITHUB_OUTPUT
+          fi
+
+          if [[ -f pr/benchmark/summary.md ]]
+          then
+            cat pr/benchmark/summary.md >> $GITHUB_OUTPUT
+          fi
+
+          echo 'EOF' >> $GITHUB_OUTPUT
+
+      - name: Create or update comment
+        if: steps.generate-comment.outputs.comment
         uses: thollander/actions-comment-pull-request@v2
         with:
-          pr_number: ${{ steps.result.outputs.pr-number }}
+          pr_number: ${{ steps.pr-number.outputs.pr-number }}
-          filePath: ecosystem-result
+          message: ${{ steps.generate-comment.outputs.comment }}
-          comment_tag: ecosystem-results
+          comment_tag: PR Check Results