diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 1d49332d04..349269a062 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -319,8 +319,8 @@ jobs:
         if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS != 'true' }}
         run: mkdocs build --strict -f mkdocs.generated.yml
 
-  check-formatter-stability:
-    name: "Check formatter stability"
+  check-formatter-ecosystem:
+    name: "Formatter ecosystem and progress checks"
     runs-on: ubuntu-latest
     needs: determine_changes
     if: needs.determine_changes.outputs.formatter == 'true'
@@ -330,7 +330,12 @@ jobs:
         run: rustup show
       - name: "Cache rust"
         uses: Swatinem/rust-cache@v2
+      - name: "Formatter progress"
+        run: scripts/formatter_progress.sh
+      - name: "Github step summary"
+        run: grep "similarity index" target/progress_projects_report.txt | sort > $GITHUB_STEP_SUMMARY
+      # CPython is not black formatted, so we run only the stability check
       - name: "Clone CPython 3.10"
         run: git clone --branch 3.10 --depth 1 https://github.com/python/cpython.git crates/ruff/resources/test/cpython
-      - name: "Check stability"
+      - name: "Check CPython stability"
         run: cargo run --bin ruff_dev -- format-dev --stability-check crates/ruff/resources/test/cpython
diff --git a/crates/ruff_dev/src/format_dev.rs b/crates/ruff_dev/src/format_dev.rs
index 6a97a627f6..af8f3a0aec 100644
--- a/crates/ruff_dev/src/format_dev.rs
+++ b/crates/ruff_dev/src/format_dev.rs
@@ -255,7 +255,7 @@ fn format_dev_multi_project(args: &Args) -> anyhow::Result<bool> {
 
                 bar.suspend(|| {
                     println!(
-                        "Finished {} with {} files (similarity index {:.3}) in {:.2}s",
+                        "Finished {}: {} files, similarity index {:.3}, {:.2}s",
                         project_path.display(),
                         result.file_count,
                         result.statistics.similarity_index(),
@@ -275,6 +275,7 @@ fn format_dev_multi_project(args: &Args) -> anyhow::Result<bool> {
                 bar.inc(1);
             }
         }
+        bar.finish_and_clear();
     }
 
     let duration = start.elapsed();
diff --git a/scripts/formatter_progress.sh b/scripts/formatter_progress.sh
new file mode 100755
index 0000000000..22031048cc
--- /dev/null
+++ b/scripts/formatter_progress.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# The pinned revisions are the latest of this writing, update freely
+
+set -ex
+
+target=$(git rev-parse --show-toplevel)/target
+dir="$target/progress_projects"
+mkdir -p "$dir"
+
+# small util library
+if [ ! -d "$dir/build" ]; then
+  git clone --filter=tree:0 https://github.com/pypa/build "$dir/build"
+  git -C "$dir/build" checkout d90f9ac6503a40ddbfaef94b7a7040f87178a4b3
+fi
+# web framework that implements a lot of magic
+if [ ! -d "$dir/django" ]; then
+  git clone --filter=tree:0 https://github.com/django/django "$dir/django"
+  git -C "$dir/django" checkout 95e4d6b81312fdd9f8ebf3385be1c1331168b5cf
+fi
+# an ML project
+if [ ! -d "$dir/transformers" ]; then
+  git clone --filter=tree:0 https://github.com/huggingface/transformers "$dir/transformers"
+  git -C "$dir/transformers" checkout c9a82be592ca305180a7ab6a36e884bca1d426b8
+fi
+# type annotations
+if [ ! -d "$dir/typeshed" ]; then
+  git clone --filter=tree:0 https://github.com/python/typeshed "$dir/typeshed"
+  git -C "$dir/typeshed" checkout 7d33060e6ae3ebe54462a891f0c566c97371915b
+fi
+# python 3.11, typing and 100% test coverage
+if [ ! -d "$dir/warehouse" ]; then
+  git clone --filter=tree:0 https://github.com/pypi/warehouse "$dir/warehouse"
+  git -C "$dir/warehouse" checkout fe6455c0a946e81f61d72edc1049f536d8bba903
+fi
+# django project
+if [ ! -d "$dir/zulip" ]; then
+  git clone --filter=tree:0 https://github.com/zulip/zulip "$dir/zulip"
+  git -C "$dir/zulip" checkout 6cb080c4479546a7f5cb017fcddea56605910b48
+fi
+
+# Uncomment if you want to update the hashes
+# for i in "$dir"/*/; do git -C "$i" switch main && git -C "$i" pull && echo "# $(basename "$i") $(git -C "$i" rev-parse HEAD)"; done
+
+time cargo run --bin ruff_dev -- format-dev --stability-check --error-file "$target/progress_projects_errors.txt" \
+  --multi-project "$dir" >"$target/progress_projects_report.txt"
+grep "similarity index" "$target/progress_projects_report.txt" | sort