From 8a7dcb794bf700f6ea230b31831e05696a54d8e2 Mon Sep 17 00:00:00 2001
From: konsti <konstin@mailbox.org>
Date: Mon, 24 Jul 2023 11:12:42 +0200
Subject: [PATCH] Add formatter progress tracking to CI (#5919)

**Summary** Add a formatter progress testing script to CI. This script
will 1) print the black compability on each run 2) catch regressions wrt
to formatter stability, emitting invalid syntax and other kinds of bugs
(e.g. #5917) before they land on main 3) have an additional layer of
real world tests when implementing new nodes or other new formatter
code.

This is currently a bash script, i'm not sure if we want to keep it that
way, or switch to e.g. the regular ecosystem scripts. The output
separation of `format_dev` could also use some polishing. We should also
consider pinning commits so we don't get spurious regression when they
change their code.

**Test Plan** The script extends CI.
---
 .github/workflows/ci.yaml         | 11 ++++++--
 crates/ruff_dev/src/format_dev.rs |  3 +-
 scripts/formatter_progress.sh     | 47 +++++++++++++++++++++++++++++++
 3 files changed, 57 insertions(+), 4 deletions(-)
 create mode 100755 scripts/formatter_progress.sh

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 1d49332d04..349269a062 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -319,8 +319,8 @@ jobs:
         if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS != 'true' }}
         run: mkdocs build --strict -f mkdocs.generated.yml
 
-  check-formatter-stability:
-    name: "Check formatter stability"
+  check-formatter-ecosystem:
+    name: "Formatter ecosystem and progress checks"
     runs-on: ubuntu-latest
     needs: determine_changes
     if: needs.determine_changes.outputs.formatter == 'true'
@@ -330,7 +330,12 @@ jobs:
         run: rustup show
       - name: "Cache rust"
         uses: Swatinem/rust-cache@v2
+      - name: "Formatter progress"
+        run: scripts/formatter_progress.sh
+      - name: "Github step summary"
+        run: grep "similarity index" target/progress_projects_report.txt | sort > $GITHUB_STEP_SUMMARY
+      # CPython is not black formatted, so we run only the stability check
       - name: "Clone CPython 3.10"
         run: git clone --branch 3.10 --depth 1 https://github.com/python/cpython.git crates/ruff/resources/test/cpython
-      - name: "Check stability"
+      - name: "Check CPython stability"
         run: cargo run --bin ruff_dev -- format-dev --stability-check crates/ruff/resources/test/cpython
diff --git a/crates/ruff_dev/src/format_dev.rs b/crates/ruff_dev/src/format_dev.rs
index 6a97a627f6..af8f3a0aec 100644
--- a/crates/ruff_dev/src/format_dev.rs
+++ b/crates/ruff_dev/src/format_dev.rs
@@ -255,7 +255,7 @@ fn format_dev_multi_project(args: &Args) -> anyhow::Result<bool> {
 
                 bar.suspend(|| {
                     println!(
-                        "Finished {} with {} files (similarity index {:.3}) in {:.2}s",
+                        "Finished {}: {} files, similarity index {:.3}, {:.2}s",
                         project_path.display(),
                         result.file_count,
                         result.statistics.similarity_index(),
@@ -275,6 +275,7 @@ fn format_dev_multi_project(args: &Args) -> anyhow::Result<bool> {
                 bar.inc(1);
             }
         }
+        bar.finish_and_clear();
     }
 
     let duration = start.elapsed();
diff --git a/scripts/formatter_progress.sh b/scripts/formatter_progress.sh
new file mode 100755
index 0000000000..22031048cc
--- /dev/null
+++ b/scripts/formatter_progress.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# The pinned revisions are the latest of this writing, update freely
+
+set -ex
+
+target=$(git rev-parse --show-toplevel)/target
+dir="$target/progress_projects"
+mkdir -p "$dir"
+
+# small util library
+if [ ! -d "$dir/build" ]; then
+  git clone --filter=tree:0 https://github.com/pypa/build "$dir/build"
+  git -C "$dir/build" checkout d90f9ac6503a40ddbfaef94b7a7040f87178a4b3
+fi
+# web framework that implements a lot of magic
+if [ ! -d "$dir/django" ]; then
+  git clone --filter=tree:0 https://github.com/django/django "$dir/django"
+  git -C "$dir/django" checkout 95e4d6b81312fdd9f8ebf3385be1c1331168b5cf
+fi
+# an ML project
+if [ ! -d "$dir/transformers" ]; then
+  git clone --filter=tree:0 https://github.com/huggingface/transformers "$dir/transformers"
+  git -C "$dir/transformers" checkout c9a82be592ca305180a7ab6a36e884bca1d426b8
+fi
+# type annotations
+if [ ! -d "$dir/typeshed" ]; then
+  git clone --filter=tree:0 https://github.com/python/typeshed "$dir/typeshed"
+  git -C "$dir/typeshed" checkout 7d33060e6ae3ebe54462a891f0c566c97371915b
+fi
+# python 3.11, typing and 100% test coverage
+if [ ! -d "$dir/warehouse" ]; then
+  git clone --filter=tree:0 https://github.com/pypi/warehouse "$dir/warehouse"
+  git -C "$dir/warehouse" checkout fe6455c0a946e81f61d72edc1049f536d8bba903
+fi
+# django project
+if [ ! -d "$dir/zulip" ]; then
+  git clone --filter=tree:0 https://github.com/zulip/zulip "$dir/zulip"
+  git -C "$dir/zulip" checkout 6cb080c4479546a7f5cb017fcddea56605910b48
+fi
+
+# Uncomment if you want to update the hashes
+# for i in "$dir"/*/; do git -C "$i" switch main && git -C "$i" pull && echo "# $(basename "$i") $(git -C "$i" rev-parse HEAD)"; done
+
+time cargo run --bin ruff_dev -- format-dev --stability-check --error-file "$target/progress_projects_errors.txt" \
+  --multi-project "$dir" >"$target/progress_projects_report.txt"
+grep "similarity index" "$target/progress_projects_report.txt" | sort