Create a separate dev crate for development scripts (#607)

ruff_dev/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "ruff_dev"
+version = "0.0.101"
+edition = "2021"
+
+[dependencies]
+anyhow = { version = "1.0.60" }
+clap = { version = "4.0.1", features = ["derive"] }
+codegen = { version = "0.2.0" }
+itertools = { version = "0.10.5" }
+ruff = { path = ".." }
+rustpython-ast = { features = ["unparse"], git = "https://github.com/RustPython/RustPython.git", rev = "77b821a1941019fe34f73ce17cea013ae1b98fd0" }
+rustpython-common = { git = "https://github.com/RustPython/RustPython.git", rev = "77b821a1941019fe34f73ce17cea013ae1b98fd0" }
+rustpython-parser = { features = ["lalrpop"], git = "https://github.com/RustPython/RustPython.git", rev = "77b821a1941019fe34f73ce17cea013ae1b98fd0" }
+strum = { version = "0.24.1", features = ["strum_macros"] }
+strum_macros = { version = "0.24.3" }

ruff_dev/src/generate_check_code_prefix.rs

@@ -0,0 +1,140 @@
+//! Generate the `CheckCodePrefix` enum.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::fs::OpenOptions;
+use std::io::Write;
+
+use anyhow::Result;
+use clap::Parser;
+use codegen::{Scope, Type, Variant};
+use itertools::Itertools;
+use ruff::checks::CheckCode;
+use strum::IntoEnumIterator;
+
+const FILE: &str = "src/checks_gen.rs";
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+pub struct Cli {
+    /// Write the generated source code to stdout (rather than to
+    /// `src/checks_gen.rs`).
+    #[arg(long)]
+    dry_run: bool,
+}
+
+pub fn main(cli: &Cli) -> Result<()> {
+    // Build up a map from prefix to matching CheckCodes.
+    let mut prefix_to_codes: BTreeMap<String, BTreeSet<CheckCode>> = Default::default();
+    for check_code in CheckCode::iter() {
+        let as_ref: String = check_code.as_ref().to_string();
+        let prefix_len = as_ref
+            .chars()
+            .take_while(|char| char.is_alphabetic())
+            .count();
+        for i in prefix_len..=as_ref.len() {
+            let prefix = as_ref[..i].to_string();
+            let entry = prefix_to_codes.entry(prefix).or_default();
+            entry.insert(check_code.clone());
+        }
+    }
+
+    let mut scope = Scope::new();
+
+    // Create the `CheckCodePrefix` definition.
+    let mut gen = scope
+        .new_enum("CheckCodePrefix")
+        .vis("pub")
+        .derive("EnumString")
+        .derive("Debug")
+        .derive("PartialEq")
+        .derive("Eq")
+        .derive("PartialOrd")
+        .derive("Ord")
+        .derive("Clone")
+        .derive("Serialize")
+        .derive("Deserialize");
+    for prefix in prefix_to_codes.keys() {
+        gen = gen.push_variant(Variant::new(prefix.to_string()));
+    }
+
+    // Create the `PrefixSpecificity` definition.
+    scope
+        .new_enum("PrefixSpecificity")
+        .vis("pub")
+        .derive("PartialEq")
+        .derive("Eq")
+        .derive("PartialOrd")
+        .derive("Ord")
+        .push_variant(Variant::new("Category"))
+        .push_variant(Variant::new("Hundreds"))
+        .push_variant(Variant::new("Tens"))
+        .push_variant(Variant::new("Explicit"));
+
+    // Create the `match` statement, to map from definition to relevant codes.
+    let mut gen = scope
+        .new_impl("CheckCodePrefix")
+        .new_fn("codes")
+        .arg_ref_self()
+        .ret(Type::new("Vec<CheckCode>"))
+        .vis("pub")
+        .line("match self {");
+    for (prefix, codes) in &prefix_to_codes {
+        gen = gen.line(format!(
+            "CheckCodePrefix::{prefix} => vec![{}],",
+            codes
+                .iter()
+                .map(|code| format!("CheckCode::{}", code.as_ref()))
+                .join(", ")
+        ));
+    }
+    gen.line("}");
+
+    // Create the `match` statement, to map from definition to specificity.
+    let mut gen = scope
+        .new_impl("CheckCodePrefix")
+        .new_fn("specificity")
+        .arg_ref_self()
+        .ret(Type::new("PrefixSpecificity"))
+        .vis("pub")
+        .line("match self {");
+    for prefix in prefix_to_codes.keys() {
+        let num_numeric = prefix.chars().filter(|char| char.is_numeric()).count();
+        let specificity = match num_numeric {
+            3 => "Explicit",
+            2 => "Tens",
+            1 => "Hundreds",
+            0 => "Category",
+            _ => panic!("Invalid prefix: {}", prefix),
+        };
+        gen = gen.line(format!(
+            "CheckCodePrefix::{prefix} => PrefixSpecificity::{},",
+            specificity
+        ));
+    }
+    gen.line("}");
+
+    // Construct the output contents.
+    let mut output = String::new();
+    output.push_str("//! File automatically generated by examples/generate_check_code_prefix.rs.");
+    output.push('\n');
+    output.push('\n');
+    output.push_str("use serde::{{Serialize, Deserialize}};");
+    output.push('\n');
+    output.push_str("use strum_macros::EnumString;");
+    output.push('\n');
+    output.push('\n');
+    output.push_str("use crate::checks::CheckCode;");
+    output.push('\n');
+    output.push('\n');
+    output.push_str(&scope.to_string());
+
+    // Write the output to `src/checks_gen.rs` (or stdout).
+    if cli.dry_run {
+        println!("{}", output);
+    } else {
+        let mut f = OpenOptions::new().write(true).truncate(true).open(FILE)?;
+        write!(f, "{}", output)?;
+    }
+
+    Ok(())
+}

ruff_dev/src/generate_rules_table.rs

@@ -0,0 +1,79 @@
+//! Generate a Markdown-compatible table of supported lint rules.
+
+use std::fs;
+use std::fs::OpenOptions;
+use std::io::Write;
+
+use anyhow::Result;
+use clap::Args;
+use ruff::checks::{CheckCategory, CheckCode};
+use strum::IntoEnumIterator;
+
+const FILE: &str = "../README.md";
+const BEGIN_PRAGMA: &str = "<!-- Begin auto-generated sections. -->";
+const END_PRAGMA: &str = "<!-- End auto-generated sections. -->";
+
+#[derive(Args)]
+pub struct Cli {
+    /// Write the generated table to stdout (rather than to `README.md`).
+    #[arg(long)]
+    dry_run: bool,
+}
+
+pub fn main(cli: &Cli) -> Result<()> {
+    // Generate the table string.
+    let mut output = String::new();
+    for check_category in CheckCategory::iter() {
+        output.push_str(&format!("### {}", check_category.title()));
+        output.push('\n');
+        output.push('\n');
+
+        output.push_str("| Coade | Name | Message | Fix |");
+        output.push('\n');
+        output.push_str("| ---- | ---- | ------- | --- |");
+        output.push('\n');
+
+        for check_code in CheckCode::iter() {
+            if check_code.category() == check_category {
+                let check_kind = check_code.kind();
+                let fix_token = if check_kind.fixable() { "🛠" } else { "" };
+                output.push_str(&format!(
+                    "| {} | {} | {} | {} |",
+                    check_kind.code().as_ref(),
+                    check_kind.as_ref(),
+                    check_kind.summary().replace('|', r"\|"),
+                    fix_token
+                ));
+                output.push('\n');
+            }
+        }
+        output.push('\n');
+    }
+
+    if cli.dry_run {
+        print!("{}", output);
+    } else {
+        // Read the existing file.
+        let existing = fs::read_to_string(FILE)?;
+
+        // Extract the prefix.
+        let index = existing
+            .find(BEGIN_PRAGMA)
+            .expect("Unable to find begin pragma.");
+        let prefix = &existing[..index + BEGIN_PRAGMA.len()];
+
+        // Extract the suffix.
+        let index = existing
+            .find(END_PRAGMA)
+            .expect("Unable to find end pragma.");
+        let suffix = &existing[index..];
+
+        // Write the prefix, new contents, and suffix.
+        let mut f = OpenOptions::new().write(true).truncate(true).open(FILE)?;
+        write!(f, "{}\n\n", prefix)?;
+        write!(f, "{}", output)?;
+        write!(f, "{}", suffix)?;
+    }
+
+    Ok(())
+}

ruff_dev/src/generate_source_code.rs

@@ -0,0 +1,25 @@
+//! Run round-trip source code generation on a given Python file.
+
+use std::path::PathBuf;
+
+use anyhow::Result;
+use clap::Args;
+use ruff::code_gen::SourceGenerator;
+use ruff::fs;
+use rustpython_parser::parser;
+
+#[derive(Args)]
+pub struct Cli {
+    /// Python file to round-trip.
+    #[arg(required = true)]
+    file: PathBuf,
+}
+
+pub fn main(cli: &Cli) -> Result<()> {
+    let contents = fs::read_file(&cli.file)?;
+    let python_ast = parser::parse_program(&contents, &cli.file.to_string_lossy())?;
+    let mut generator = SourceGenerator::new();
+    generator.unparse_suite(&python_ast)?;
+    println!("{}", generator.generate()?);
+    Ok(())
+}

ruff_dev/src/lib.rs

@@ -0,0 +1,5 @@
+pub mod generate_check_code_prefix;
+pub mod generate_rules_table;
+pub mod generate_source_code;
+pub mod print_ast;
+pub mod print_tokens;

ruff_dev/src/main.rs

@@ -0,0 +1,39 @@
+use anyhow::Result;
+use clap::{Parser, Subcommand};
+use ruff_dev::{
+    generate_check_code_prefix, generate_rules_table, generate_source_code, print_ast, print_tokens,
+};
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+#[command(propagate_version = true)]
+struct Cli {
+    #[command(subcommand)]
+    command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+    /// Generate the `CheckCodePrefix` enum.
+    GenerateCheckCodePrefix(generate_check_code_prefix::Cli),
+    /// Generate a Markdown-compatible table of supported lint rules.
+    GenerateRulesTable(generate_rules_table::Cli),
+    /// Run round-trip source code generation on a given Python file.
+    GenerateSourceCode(generate_source_code::Cli),
+    /// Print the AST for a given Python file.
+    PrintAST(print_ast::Cli),
+    /// Print the token stream for a given Python file.
+    PrintTokens(print_tokens::Cli),
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+    match &cli.command {
+        Commands::GenerateCheckCodePrefix(args) => generate_check_code_prefix::main(args)?,
+        Commands::GenerateRulesTable(args) => generate_rules_table::main(args)?,
+        Commands::GenerateSourceCode(args) => generate_source_code::main(args)?,
+        Commands::PrintAST(args) => print_ast::main(args)?,
+        Commands::PrintTokens(args) => print_tokens::main(args)?,
+    }
+    Ok(())
+}

ruff_dev/src/print_ast.rs

@@ -0,0 +1,22 @@
+//! Print the AST for a given Python file.
+
+use std::path::PathBuf;
+
+use anyhow::Result;
+use clap::Args;
+use ruff::fs;
+use rustpython_parser::parser;
+
+#[derive(Args)]
+pub struct Cli {
+    /// Python file for which to generate the AST.
+    #[arg(required = true)]
+    file: PathBuf,
+}
+
+pub fn main(cli: &Cli) -> Result<()> {
+    let contents = fs::read_file(&cli.file)?;
+    let python_ast = parser::parse_program(&contents, &cli.file.to_string_lossy())?;
+    println!("{:#?}", python_ast);
+    Ok(())
+}

ruff_dev/src/print_tokens.rs

@@ -0,0 +1,23 @@
+//! Print the token stream for a given Python file.
+
+use std::path::PathBuf;
+
+use anyhow::Result;
+use clap::Args;
+use ruff::fs;
+use rustpython_parser::lexer;
+
+#[derive(Args)]
+pub struct Cli {
+    /// Python file for which to generate the AST.
+    #[arg(required = true)]
+    file: PathBuf,
+}
+
+pub fn main(cli: &Cli) -> Result<()> {
+    let contents = fs::read_file(&cli.file)?;
+    for (_, tok, _) in lexer::make_tokenizer(&contents).flatten() {
+        println!("{:#?}", tok);
+    }
+    Ok(())
+}