From aa51ecedc5ebbba5173350d45f0ea4ae9ae41df6 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Thu, 16 Mar 2023 09:05:10 +0100 Subject: [PATCH] ci: Benchmark CI Step (#3480) --- .github/workflows/benchmark.yaml | 133 ++++++++++++++++++++++++++++++ .github/workflows/ci.yaml | 19 ++++- .github/workflows/pr-comment.yaml | 69 +++++++++++++--- 3 files changed, 206 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/benchmark.yaml diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 0000000000..48bd53fa64 --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,133 @@ +name: Benchmark + +on: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + run-benchmark: + if: github.event_name == 'pull_request' + name: "Run | ${{ matrix.os }}" + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + + steps: + - name: "PR - Checkout Branch" + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "PR - Install Rust toolchain" + run: rustup show + + - uses: Swatinem/rust-cache@v1 + + - name: "PR - Build benchmarks" + uses: actions-rs/cargo@v1 + with: + command: bench + args: -p ruff_benchmark --no-run + + - name: "PR - Run benchmarks" + run: cargo benchmark --save-baseline=pr + + - name: "Main - Checkout Branch" + uses: actions/checkout@v3 + with: + clean: false + ref: main + + - name: "Main - Install Rust toolchain" + run: rustup show + + - name: "Main - Build benchmarks" + uses: actions-rs/cargo@v1 + with: + command: bench + args: -p ruff_benchmark --no-run + + - name: "Main - Run benchmarks" + run: cargo benchmark --save-baseline=main + + - name: "Upload benchmark results" + uses: actions/upload-artifact@v3 + with: + name: benchmark-results-${{ matrix.os }} + path: ./target/criterion + + # Cleanup + - name: Remove Criterion Artifact + uses: JesseTG/rm@v1.0.3 + with: + path: ./target/criterion + + benchmark-compare: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + name: Compare + needs: + - run-benchmark + + steps: + - name: "Install Rust toolchain" + run: rustup show + + - name: "Install critcmp" + # Use debug build: Building takes much longer than the "slowness" of using the debug build. + run: cargo install --debug critcmp + + - name: "Linux | Download PR benchmark results" + uses: actions/download-artifact@v3 + with: + name: benchmark-results-ubuntu-latest + path: ./target/criterion + + - name: "Linux | Compare benchmark results" + shell: bash + run: | + echo "### Benchmark" >> summary.md + echo "#### Linux" >> summary.md + echo "\`\`\`" >> summary.md + critcmp main pr >> summary.md + echo "\`\`\`" >> summary.md + echo "" >> summary.md + + - name: "Linux | Cleanup benchmark results" + run: rm -rf ./target/criterion + + - name: "Windows | Download PR benchmark results" + uses: actions/download-artifact@v3 + with: + name: benchmark-results-windows-latest + path: ./target/criterion + + - name: "Windows | Compare benchmark results" + shell: bash + run: | + echo "#### Windows" >> summary.md + echo "\`\`\`" >> summary.md + critcmp main pr >> summary.md + echo "\`\`\`" >> summary.md + echo "" >> summary.md + + echo ${{ github.event.pull_request.number }} > pr-number + + cat summary.md > $GITHUB_STEP_SUMMARY + + - uses: actions/upload-artifact@v3 + name: Upload PR Number + with: + name: pr-number + path: pr-number + + - uses: actions/upload-artifact@v3 + name: Upload Summary + with: + name: summary + path: summary.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1c48a0c44a..1e710e7ac3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -80,6 +80,7 @@ jobs: # Setting RUSTDOCFLAGS because `cargo doc --check` isn't yet implemented (https://github.com/rust-lang/cargo/issues/10025). RUSTDOCFLAGS: "-D warnings" - uses: actions/upload-artifact@v3 + if: ${{ matrix.os == 'ubuntu-latest' }} with: name: ruff path: target/debug/ruff @@ -139,27 +140,39 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.11" + - uses: actions/download-artifact@v3 + name: Download Ruff binary id: ruff-target with: name: ruff path: target/debug + - uses: dawidd6/action-download-artifact@v2 + name: Download base results with: name: ruff branch: ${{ github.event.pull_request.base.ref }} check_artifacts: true + - name: Run ecosystem check run: | # Make executable, since artifact download doesn't preserve this chmod +x ruff ${{ steps.ruff-target.outputs.download-path }}/ruff scripts/check_ecosystem.py ruff ${{ steps.ruff-target.outputs.download-path }}/ruff | tee ecosystem-result + cat ecosystem-result > $GITHUB_STEP_SUMMARY echo ${{ github.event.number }} > pr-number + - uses: actions/upload-artifact@v3 + name: Upload PR Number + with: + name: pr-number + path: pr-number + + - uses: actions/upload-artifact@v3 + name: Upload Results with: name: ecosystem-result - path: | - ecosystem-result - pr-number + path: ecosystem-result diff --git a/.github/workflows/pr-comment.yaml b/.github/workflows/pr-comment.yaml index c9f652bdd0..80ecbd9c14 100644 --- a/.github/workflows/pr-comment.yaml +++ b/.github/workflows/pr-comment.yaml @@ -2,7 +2,7 @@ name: PR Check Comment on: workflow_run: - workflows: [CI] + workflows: [CI, Benchmark] types: [completed] workflow_dispatch: inputs: @@ -17,22 +17,67 @@ jobs: comment: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - uses: dawidd6/action-download-artifact@v2 - id: download-result + name: Download PR Number + with: + name: pr-number + run_id: ${{ github.event.workflow_run.id || github.event.inputs.workflow_run_id }} + if_no_artifact_found: ignore + + - name: Extract PR Number + id: pr-number + run: | + if [[ -f pr-number ]] + then + echo "pr-number=$(> $GITHUB_OUTPUT + fi + + - uses: dawidd6/action-download-artifact@v2 + name: "Download Ecosystem Result" + id: download-ecosystem-result + if: steps.pr-number.outputs.pr-number with: name: ecosystem-result workflow: ci.yaml - run_id: ${{ github.event.workflow_run.id || github.event.inputs.workflow_run_id }} + pr: ${{ steps.pr-number.outputs.pr-number }} + path: pr/ecosystem if_no_artifact_found: ignore - - if: steps.download-result.outputs.found_artifact - id: result + + - uses: dawidd6/action-download-artifact@v2 + name: "Download Benchmark Result" + id: download-benchmark-result + if: steps.pr-number.outputs.pr-number + with: + name: summary + workflow: benchmark.yaml + pr: ${{ steps.pr-number.outputs.pr-number }} + path: pr/benchmark + if_no_artifact_found: ignore + + - name: Generate Comment + id: generate-comment + if: steps.download-ecosystem-result.outputs.found_artifact == 'true' || steps.download-benchmark-result.outputs.found_artifact == 'true' run: | - echo "pr-number=$(> $GITHUB_OUTPUT - - name: Create comment - if: steps.download-result.outputs.found_artifact + echo 'comment<> $GITHUB_OUTPUT + echo '## PR Check Results' >> $GITHUB_OUTPUT + + if [[ -f pr/ecosystem/ecosystem-result ]] + then + echo "### Ecosystem" >> $GITHUB_OUTPUT + cat pr/ecosystem/ecosystem-result >> $GITHUB_OUTPUT + fi + + if [[ -f pr/benchmark/summary.md ]] + then + cat pr/benchmark/summary.md >> $GITHUB_OUTPUT + fi + + echo 'EOF' >> $GITHUB_OUTPUT + + - name: Create or update comment + if: steps.generate-comment.outputs.comment uses: thollander/actions-comment-pull-request@v2 with: - pr_number: ${{ steps.result.outputs.pr-number }} - filePath: ecosystem-result - comment_tag: ecosystem-results + pr_number: ${{ steps.pr-number.outputs.pr-number }} + message: ${{ steps.generate-comment.outputs.comment }} + comment_tag: PR Check Results