diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 25025e8ac8..71ccf1bcbe 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -55,6 +55,7 @@ jobs:
               - crates/ruff_python_index/**
               - crates/ruff_text_size/**
               - crates/ruff_python_parser/**
+              - crates/ruff_dev/**
 
   cargo-fmt:
     name: "cargo fmt"
@@ -336,7 +337,7 @@ jobs:
       - name: "Formatter progress"
         run: scripts/formatter_ecosystem_checks.sh
       - name: "Github step summary"
-        run: grep "similarity index" target/progress_projects_report.txt | sort > $GITHUB_STEP_SUMMARY
+        run: grep "similarity index" target/progress_projects_log.txt | sort > $GITHUB_STEP_SUMMARY
       # CPython is not black formatted, so we run only the stability check
       - name: "Clone CPython 3.10"
         run: git clone --branch 3.10 --depth 1 https://github.com/python/cpython.git crates/ruff/resources/test/cpython
diff --git a/crates/ruff_dev/src/format_dev.rs b/crates/ruff_dev/src/format_dev.rs
index 149f0c7faf..a669ed022c 100644
--- a/crates/ruff_dev/src/format_dev.rs
+++ b/crates/ruff_dev/src/format_dev.rs
@@ -184,12 +184,19 @@ pub(crate) struct Args {
     /// Write all errors to this file in addition to stdout. Only used in multi-project mode.
     #[arg(long)]
     pub(crate) error_file: Option<PathBuf>,
+    /// Write all log messages (same as cli) to this file
+    #[arg(long)]
+    pub(crate) log_file: Option<PathBuf>,
+    /// Assert that there are exactly this many input files with errors. This catches regressions
+    /// (or improvements) in the parser.
+    #[arg(long)]
+    pub(crate) files_with_errors: Option<u32>,
     #[clap(flatten)]
     pub(crate) log_level_args: LogLevelArgs,
 }
 
 pub(crate) fn main(args: &Args) -> anyhow::Result<ExitCode> {
-    setup_logging(&args.log_level_args);
+    setup_logging(&args.log_level_args, args.log_file.as_deref())?;
 
     let all_success = if args.multi_project {
         format_dev_multi_project(args)?
@@ -202,13 +209,24 @@ pub(crate) fn main(args: &Args) -> anyhow::Result<ExitCode> {
         }
         info!(
             parent: None,
-            "Found {} stability errors in {} files (similarity index {:.3}) in {:.2}s",
+            "Done: {} stability errors, {} files, similarity index {:.3}), took {:.2}s, {} input files contained syntax errors ",
             error_count,
             result.file_count,
             result.statistics.similarity_index(),
             result.duration.as_secs_f32(),
+            result.syntax_error_in_input,
         );
 
+        if let Some(files_with_errors) = args.files_with_errors {
+            if result.syntax_error_in_input != files_with_errors {
+                error!(
+                    "Expected {files_with_errors} input files with errors, found {}",
+                    result.syntax_error_in_input
+                );
+                return Ok(ExitCode::FAILURE);
+            }
+        }
+
         error_count == 0
     };
     if all_success {
@@ -218,7 +236,7 @@ pub(crate) fn main(args: &Args) -> anyhow::Result<ExitCode> {
     }
 }
 
-fn setup_logging(log_level_args: &LogLevelArgs) {
+fn setup_logging(log_level_args: &LogLevelArgs, log_file: Option<&Path>) -> io::Result<()> {
     // Custom translation since we need the tracing type for `EnvFilter`
     let log_level = match LogLevel::from(log_level_args) {
         LogLevel::Default => tracing::Level::INFO,
@@ -236,17 +254,25 @@ fn setup_logging(log_level_args: &LogLevelArgs) {
     let indicitif_compatible_writer_layer = tracing_subscriber::fmt::layer()
         .with_writer(indicatif_layer.get_stderr_writer())
         .with_target(false);
+    let log_layer = log_file.map(File::create).transpose()?.map(|log_file| {
+        tracing_subscriber::fmt::layer()
+            .with_writer(log_file)
+            .with_ansi(false)
+    });
     tracing_subscriber::registry()
         .with(filter_layer)
         .with(indicitif_compatible_writer_layer)
         .with(indicatif_layer)
+        .with(log_layer)
         .init();
+    Ok(())
 }
 
 /// Checks a directory of projects
 fn format_dev_multi_project(args: &Args) -> anyhow::Result<bool> {
     let mut total_errors = 0;
     let mut total_files = 0;
+    let mut total_syntax_error_in_input = 0;
     let start = Instant::now();
 
     let mut project_paths = Vec::new();
@@ -277,20 +303,23 @@ fn format_dev_multi_project(args: &Args) -> anyhow::Result<bool> {
     };
 
     for project_path in project_paths {
-        info!(parent: None, "Starting {}", project_path.display());
+        debug!(parent: None, "Starting {}", project_path.display());
 
         match format_dev_project(&[project_path.clone()], args.stability_check, args.write) {
             Ok(result) => {
                 total_errors += result.error_count();
                 total_files += result.file_count;
+                total_syntax_error_in_input += result.syntax_error_in_input;
 
                 info!(
                     parent: None,
-                    "Finished {}: {} files, similarity index {:.3}, {:.2}s",
+                    "Finished {}: {} stability errors, {} files, similarity index {:.3}), took {:.2}s, {} input files contained syntax errors ",
                     project_path.display(),
+                    result.error_count(),
                     result.file_count,
                     result.statistics.similarity_index(),
                     result.duration.as_secs_f32(),
+                    result.syntax_error_in_input,
                 );
                 if result.error_count() > 0 {
                     error!(
@@ -320,10 +349,20 @@ fn format_dev_multi_project(args: &Args) -> anyhow::Result<bool> {
 
     info!(
         parent: None,
-        "{total_errors} stability errors in {total_files} files in {}s",
-        duration.as_secs_f32()
+        "Finished: {total_errors} stability errors, {total_files} files, tool {}s, {total_syntax_error_in_input} input files contained syntax errors ",
+        duration.as_secs_f32(),
     );
 
+    if let Some(files_with_errors) = args.files_with_errors {
+        if total_syntax_error_in_input != files_with_errors {
+            error!(
+                "Expected {files_with_errors} input files with errors, found {}",
+                total_syntax_error_in_input
+            );
+            return Ok(false);
+        }
+    }
+
     Ok(total_errors == 0)
 }
 
@@ -368,12 +407,27 @@ fn format_dev_project(
 
     let mut statistics = Statistics::default();
     let mut formatted_counter = 0;
+    let mut syntax_error_in_input = 0;
     let mut diagnostics = Vec::new();
     for (result, file) in results {
         formatted_counter += 1;
         match result {
             Ok(statistics_file) => statistics += statistics_file,
-            Err(error) => diagnostics.push(Diagnostic { file, error }),
+            Err(error) => {
+                match error {
+                    CheckFileError::SyntaxErrorInInput(error) => {
+                        // This is not our error
+                        debug!(
+                            parent: None,
+                            "Syntax error in {}: {}",
+                            file.display(),
+                            error
+                        );
+                        syntax_error_in_input += 1;
+                    }
+                    _ => diagnostics.push(Diagnostic { file, error }),
+                }
+            }
         }
     }
 
@@ -384,6 +438,7 @@ fn format_dev_project(
         file_count: formatted_counter,
         diagnostics,
         statistics,
+        syntax_error_in_input,
     })
 }
 
@@ -408,19 +463,7 @@ fn format_dir_entry(
     // Handle panics (mostly in `debug_assert!`)
     let result =
         match catch_unwind(|| format_dev_file(&file, stability_check, write, options.clone())) {
-            Ok(result) => match result {
-                Err(CheckFileError::SyntaxErrorInInput(error)) => {
-                    // We don't care about this error, only log it
-                    info!(
-                        parent: None,
-                        "Syntax error in {}: {}",
-                        file.display(),
-                        error
-                    );
-                    Ok(Statistics::default())
-                }
-                _ => result,
-            },
+            Ok(result) => result,
             Err(panic) => {
                 if let Some(message) = panic.downcast_ref::<String>() {
                     Err(CheckFileError::Panic {
@@ -472,6 +515,7 @@ struct CheckRepoResult {
     file_count: usize,
     diagnostics: Vec<Diagnostic>,
     statistics: Statistics,
+    syntax_error_in_input: u32,
 }
 
 impl CheckRepoResult {
diff --git a/scripts/formatter_ecosystem_checks.sh b/scripts/formatter_ecosystem_checks.sh
index b213a27fd9..c4aa2db57b 100755
--- a/scripts/formatter_ecosystem_checks.sh
+++ b/scripts/formatter_ecosystem_checks.sh
@@ -41,19 +41,24 @@ if [ ! -d "$dir/warehouse" ]; then
   git clone --filter=tree:0 https://github.com/pypi/warehouse "$dir/warehouse"
   git -C "$dir/warehouse" checkout fe6455c0a946e81f61d72edc1049f536d8bba903
 fi
-# django project
+# zulip, a django user
 if [ ! -d "$dir/zulip" ]; then
   git clone --filter=tree:0 https://github.com/zulip/zulip "$dir/zulip"
   git -C "$dir/zulip" checkout 6cb080c4479546a7f5cb017fcddea56605910b48
 fi
+# cpython itself
+if [ ! -d "$dir/cpython" ]; then
+  git clone --filter=tree:0 https://github.com/python/cpython "$dir/cpython"
+  git -C "$dir/cpython" checkout 45de31db9cc9be945702f3a7ca35bbb9f98476af
+fi
 
 # Uncomment if you want to update the hashes
 # for i in "$dir"/*/; do git -C "$i" switch main && git -C "$i" pull && echo "# $(basename "$i") $(git -C "$i" rev-parse HEAD)"; done
 
 time cargo run --bin ruff_dev -- format-dev --stability-check --error-file "$target/progress_projects_errors.txt" \
-  --multi-project "$dir" >"$target/progress_projects_report.txt" || (
+  --log-file "$target/progress_projects_log.txt" --files-with-errors 25 --multi-project "$dir" || (
   echo "Ecosystem check failed"
-  cat "$target/progress_projects_report.txt"
+  cat "$target/progress_projects_log.txt"
   exit 1
 )
-grep "similarity index" "$target/progress_projects_report.txt" | sort
+grep "similarity index" "$target/progress_projects_log.txt" | sort