diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 4e2b400b05..0985896c6c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -183,18 +183,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: "Install Rust toolchain"
-        run: rustup toolchain install nightly
+      - name: "Install nightly Rust toolchain"
+        # Only pinned to make caching work, update freely
+        run: rustup toolchain install nightly-2023-03-30
+      - uses: Swatinem/rust-cache@v2
       - name: "Install cargo-udeps"
         uses: taiki-e/install-action@cargo-udeps
       - name: "Run cargo-udeps"
         run: |
-          unused_dependencies=$(cargo +nightly udeps > unused.txt && cat unused.txt | cut -d $'\n' -f 2-)
+          unused_dependencies=$(cargo +nightly-2023-03-30 udeps > unused.txt && cat unused.txt | cut -d $'\n' -f 2-)
           if [ -z "$unused_dependencies" ]; then
             echo "No unused dependencies found" > $GITHUB_STEP_SUMMARY
             exit 0
           else
-            echo "Unused dependencies found" > $GITHUB_STEP_SUMMARY
+            echo "Found unused dependencies" > $GITHUB_STEP_SUMMARY
             echo '```console' >> $GITHUB_STEP_SUMMARY
             echo "$unused_dependencies" >> $GITHUB_STEP_SUMMARY
             echo '```' >> $GITHUB_STEP_SUMMARY