diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 505a7440ac..c654e23302 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -17,20 +17,6 @@ env:
   RUSTUP_MAX_RETRIES: 10
 
 jobs:
-  cargo-build:
-    name: "cargo build"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: "Install Rust toolchain"
-        run: rustup show
-      - uses: Swatinem/rust-cache@v1
-      - run: cargo build --all
-      - run: ./target/debug/ruff_dev generate-all
-      - run: git diff --quiet README.md || echo "::error file=README.md::This file is outdated. Run 'cargo dev generate-all'."
-      - run: git diff --quiet ruff.schema.json || echo "::error file=ruff.schema.json::This file is outdated. Run 'cargo dev generate-all'."
-      - run: git diff --exit-code -- README.md ruff.schema.json docs
-
   cargo-fmt:
     name: "cargo fmt"
     runs-on: ubuntu-latest
diff --git a/Cargo.lock b/Cargo.lock
index ba5300ad3a..8ae249830d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -554,6 +554,16 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "ctor"
+version = "0.1.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096"
+dependencies = [
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "cxx"
 version = "1.0.91"
@@ -1448,6 +1458,15 @@ version = "6.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
 
+[[package]]
+name = "output_vt100"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.12.1"
@@ -1719,6 +1738,18 @@ dependencies = [
  "termtree",
 ]
 
+[[package]]
+name = "pretty_assertions"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755"
+dependencies = [
+ "ctor",
+ "diff",
+ "output_vt100",
+ "yansi",
+]
+
 [[package]]
 name = "proc-macro-error"
 version = "1.0.4"
@@ -2051,6 +2082,7 @@ dependencies = [
  "itertools",
  "libcst",
  "once_cell",
+ "pretty_assertions",
  "regex",
  "ruff",
  "ruff_cli",
@@ -3266,6 +3298,12 @@ dependencies = [
  "linked-hash-map",
 ]
 
+[[package]]
+name = "yansi"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"
+
 [[package]]
 name = "yansi-term"
 version = "0.1.2"
diff --git a/crates/ruff/src/rule_selector.rs b/crates/ruff/src/rule_selector.rs
index 9b21d23c74..14f12ef495 100644
--- a/crates/ruff/src/rule_selector.rs
+++ b/crates/ruff/src/rule_selector.rs
@@ -184,6 +184,20 @@ impl JsonSchema for RuleSelector {
                 std::iter::once("ALL".to_string())
                     .chain(
                         RuleCodePrefix::iter()
+                            .filter(|p| {
+                                // Once logical lines are active by default, please remove this.
+                                // This is here because generate-all output otherwise depends on
+                                // the feature sets which makes the test running with
+                                // `--all-features` fail
+                                !Rule::from_code(&format!(
+                                    "{}{}",
+                                    p.linter().common_prefix(),
+                                    p.short_code()
+                                ))
+                                .unwrap()
+                                .lint_source()
+                                .is_logical_lines()
+                            })
                             .map(|p| {
                                 let prefix = p.linter().common_prefix();
                                 let code = p.short_code();
diff --git a/crates/ruff_dev/Cargo.toml b/crates/ruff_dev/Cargo.toml
index 9121bb7cbf..2731d195d3 100644
--- a/crates/ruff_dev/Cargo.toml
+++ b/crates/ruff_dev/Cargo.toml
@@ -11,6 +11,7 @@ clap = { workspace = true }
 itertools = { workspace = true }
 libcst = { workspace = true }
 once_cell = { workspace = true }
+pretty_assertions = { version = "1.3.0" }
 regex = { workspace = true }
 ruff = { path = "../ruff" }
 ruff_cli = { path = "../ruff_cli" }
diff --git a/crates/ruff_dev/src/generate_all.rs b/crates/ruff_dev/src/generate_all.rs
index 25c779e7a0..38d35ecb58 100644
--- a/crates/ruff_dev/src/generate_all.rs
+++ b/crates/ruff_dev/src/generate_all.rs
@@ -4,22 +4,32 @@ use anyhow::Result;
 
 use crate::{generate_cli_help, generate_docs, generate_json_schema};
 
+pub const REGENERATE_ALL_COMMAND: &str = "cargo dev generate-all";
+
 #[derive(clap::Args)]
 pub struct Args {
     /// Write the generated artifacts to stdout (rather than to the filesystem).
     #[arg(long)]
     dry_run: bool,
+    /// Don't write to the file, check if the file is up-to-date and error if not
+    #[arg(long)]
+    check: bool,
 }
 
 pub fn main(args: &Args) -> Result<()> {
-    generate_docs::main(&generate_docs::Args {
-        dry_run: args.dry_run,
-    })?;
+    // Not checked in
+    if !args.check {
+        generate_docs::main(&generate_docs::Args {
+            dry_run: args.dry_run,
+        })?;
+    }
     generate_json_schema::main(&generate_json_schema::Args {
         dry_run: args.dry_run,
+        check: args.check,
     })?;
     generate_cli_help::main(&generate_cli_help::Args {
         dry_run: args.dry_run,
+        check: args.check,
     })?;
     Ok(())
 }
diff --git a/crates/ruff_dev/src/generate_cli_help.rs b/crates/ruff_dev/src/generate_cli_help.rs
index a285b10b45..908a42d681 100644
--- a/crates/ruff_dev/src/generate_cli_help.rs
+++ b/crates/ruff_dev/src/generate_cli_help.rs
@@ -1,13 +1,13 @@
 //! Generate CLI help.
 #![allow(clippy::print_stdout, clippy::print_stderr)]
 
-use std::fs::OpenOptions;
-use std::io::Write;
 use std::path::PathBuf;
 use std::{fs, str};
 
-use anyhow::Result;
+use anyhow::{bail, Result};
+use pretty_assertions::StrComparison;
 
+use crate::generate_all::REGENERATE_ALL_COMMAND;
 use crate::ROOT_DIR;
 
 const COMMAND_HELP_BEGIN_PRAGMA: &str = "<!-- Begin auto-generated command help. -->\n";
@@ -21,17 +21,22 @@ pub struct Args {
     /// Write the generated help to stdout (rather than to `docs/configuration.md`).
     #[arg(long)]
     pub(crate) dry_run: bool,
+    /// Don't write to the file, check if the file is up-to-date and error if not
+    #[arg(long)]
+    pub(crate) check: bool,
 }
 
 fn trim_lines(s: &str) -> String {
     s.lines().map(str::trim_end).collect::<Vec<_>>().join("\n")
 }
 
-fn replace_docs_section(content: &str, begin_pragma: &str, end_pragma: &str) -> Result<()> {
-    // Read the existing file.
-    let file = PathBuf::from(ROOT_DIR).join("docs/configuration.md");
-    let existing = fs::read_to_string(&file)?;
-
+/// Takes the existing file contents, inserts the section, returns the transformed content
+fn replace_docs_section(
+    existing: &str,
+    section: &str,
+    begin_pragma: &str,
+    end_pragma: &str,
+) -> String {
     // Extract the prefix.
     let index = existing
         .find(begin_pragma)
@@ -44,13 +49,7 @@ fn replace_docs_section(content: &str, begin_pragma: &str, end_pragma: &str) ->
         .expect("Unable to find end pragma");
     let suffix = &existing[index..];
 
-    // Write the prefix, new contents, and suffix.
-    let mut f = OpenOptions::new().write(true).truncate(true).open(&file)?;
-    writeln!(f, "{prefix}")?;
-    write!(f, "{content}")?;
-    write!(f, "{suffix}")?;
-
-    Ok(())
+    format!("{prefix}\n{section}{suffix}")
 }
 
 pub fn main(args: &Args) -> Result<()> {
@@ -64,17 +63,49 @@ pub fn main(args: &Args) -> Result<()> {
         print!("{command_help}");
         print!("{subcommand_help}");
     } else {
-        replace_docs_section(
+        // Read the existing file.
+        let filename = "docs/configuration.md";
+        let file = PathBuf::from(ROOT_DIR).join(filename);
+        let existing = fs::read_to_string(&file)?;
+
+        let new = replace_docs_section(
+            &existing,
             &format!("```text\n{command_help}\n```\n\n"),
             COMMAND_HELP_BEGIN_PRAGMA,
             COMMAND_HELP_END_PRAGMA,
-        )?;
-        replace_docs_section(
+        );
+        let new = replace_docs_section(
+            &new,
             &format!("```text\n{subcommand_help}\n```\n\n"),
             SUBCOMMAND_HELP_BEGIN_PRAGMA,
             SUBCOMMAND_HELP_END_PRAGMA,
-        )?;
+        );
+
+        if args.check {
+            if existing == new {
+                println!("up-to-date: {filename}");
+            } else {
+                let comparison = StrComparison::new(&existing, &new);
+                bail!("{filename} changed, please run `{REGENERATE_ALL_COMMAND}`:\n{comparison}");
+            }
+        } else {
+            fs::write(file, &new)?;
+        }
     }
 
     Ok(())
 }
+
+#[cfg(test)]
+mod test {
+    use super::{main, Args};
+    use anyhow::Result;
+
+    #[test]
+    fn test_generate_json_schema() -> Result<()> {
+        main(&Args {
+            dry_run: false,
+            check: true,
+        })
+    }
+}
diff --git a/crates/ruff_dev/src/generate_docs.rs b/crates/ruff_dev/src/generate_docs.rs
index 5f613d46f5..cbe60a5954 100644
--- a/crates/ruff_dev/src/generate_docs.rs
+++ b/crates/ruff_dev/src/generate_docs.rs
@@ -2,14 +2,18 @@
 #![allow(clippy::print_stdout, clippy::print_stderr)]
 
 use std::fs;
+use std::path::PathBuf;
 
 use anyhow::Result;
 use regex::{Captures, Regex};
+use strum::IntoEnumIterator;
+
 use ruff::registry::{Linter, Rule, RuleNamespace};
 use ruff::settings::options::Options;
 use ruff::settings::options_base::ConfigurationOptions;
 use ruff::AutofixAvailability;
-use strum::IntoEnumIterator;
+
+use crate::ROOT_DIR;
 
 #[derive(clap::Args)]
 pub struct Args {
@@ -44,11 +48,17 @@ pub fn main(args: &Args) -> Result<()> {
 
             process_documentation(explanation.trim(), &mut output);
 
+            let filename = PathBuf::from(ROOT_DIR)
+                .join("docs")
+                .join("rules")
+                .join(rule.as_ref())
+                .with_extension("md");
+
             if args.dry_run {
                 println!("{output}");
             } else {
                 fs::create_dir_all("docs/rules")?;
-                fs::write(format!("docs/rules/{}.md", rule.as_ref()), output)?;
+                fs::write(filename, output)?;
             }
         }
     }
diff --git a/crates/ruff_dev/src/generate_json_schema.rs b/crates/ruff_dev/src/generate_json_schema.rs
index 6f5490d8e3..7ca4e9f056 100644
--- a/crates/ruff_dev/src/generate_json_schema.rs
+++ b/crates/ruff_dev/src/generate_json_schema.rs
@@ -3,7 +3,9 @@
 use std::fs;
 use std::path::PathBuf;
 
-use anyhow::Result;
+use crate::generate_all::REGENERATE_ALL_COMMAND;
+use anyhow::{bail, Result};
+use pretty_assertions::StrComparison;
 use ruff::settings::options::Options;
 use schemars::schema_for;
 
@@ -14,17 +16,44 @@ pub struct Args {
     /// Write the generated table to stdout (rather than to `ruff.schema.json`).
     #[arg(long)]
     pub(crate) dry_run: bool,
+    /// Don't write to the file, check if the file is up-to-date and error if not
+    #[arg(long)]
+    pub(crate) check: bool,
 }
 
 pub fn main(args: &Args) -> Result<()> {
     let schema = schema_for!(Options);
     let schema_string = serde_json::to_string_pretty(&schema).unwrap();
+    let filename = "ruff.schema.json";
+    let schema_path = PathBuf::from(ROOT_DIR).join(filename);
 
     if args.dry_run {
         println!("{schema_string}");
+    } else if args.check {
+        let current = fs::read_to_string(schema_path)?;
+        if current == schema_string {
+            println!("up-to-date: {filename}");
+        } else {
+            let comparison = StrComparison::new(&current, &schema_string);
+            bail!("{filename} changed, please run `{REGENERATE_ALL_COMMAND}`:\n{comparison}");
+        }
     } else {
-        let file = PathBuf::from(ROOT_DIR).join("ruff.schema.json");
+        let file = schema_path;
         fs::write(file, schema_string.as_bytes())?;
     }
     Ok(())
 }
+
+#[cfg(test)]
+mod test {
+    use super::{main, Args};
+    use anyhow::Result;
+
+    #[test]
+    fn test_generate_json_schema() -> Result<()> {
+        main(&Args {
+            dry_run: false,
+            check: true,
+        })
+    }
+}