diff --git a/Cargo.lock b/Cargo.lock index 8ed284d47a..646078fa99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,17 +14,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "ahash" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" -dependencies = [ - "getrandom", - "once_cell", - "version_check", -] - [[package]] name = "aho-corasick" version = "0.7.20" @@ -809,9 +798,6 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash", -] [[package]] name = "heck" @@ -1798,6 +1784,7 @@ dependencies = [ "ruff_python_stdlib", "ruff_rustpython", "ruff_text_size", + "ruff_textwrap", "rustc-hash", "rustpython-format", "rustpython-parser", @@ -1811,7 +1798,6 @@ dependencies = [ "strum", "strum_macros", "test-case", - "textwrap", "thiserror", "toml", "typed-arena", @@ -1880,13 +1866,13 @@ dependencies = [ "ruff_python_ast", "ruff_python_stdlib", "ruff_text_size", + "ruff_textwrap", "rustc-hash", "serde", "serde_json", "shellexpand", "similar", "strum", - "textwrap", "tikv-jemallocator", "ureq", "walkdir", @@ -1907,13 +1893,13 @@ dependencies = [ "ruff", "ruff_cli", "ruff_diagnostics", + "ruff_textwrap", "rustpython-format", "rustpython-parser", "schemars", "serde_json", "strum", "strum_macros", - "textwrap", ] [[package]] @@ -1956,8 +1942,8 @@ dependencies = [ "itertools", "proc-macro2", "quote", + "ruff_textwrap", "syn 2.0.15", - "textwrap", ] [[package]] @@ -2066,6 +2052,14 @@ dependencies = [ "serde", ] +[[package]] +name = "ruff_textwrap" +version = "0.0.0" +dependencies = [ + "ruff_newlines", + "ruff_text_size", +] + [[package]] name = "ruff_wasm" version = "0.0.0" @@ -2357,12 +2351,6 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" -[[package]] -name = "smawk" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f67ad224767faa3c7d8b6d91985b78e70a1324408abcb1cfcc2be4c06bc06043" - [[package]] name = "spin" version = "0.5.2" @@ -2506,11 +2494,6 @@ name = "textwrap" version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" -dependencies = [ - "smawk", - "unicode-linebreak", - "unicode-width", -] [[package]] name = "thiserror" @@ -2756,16 +2739,6 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" -[[package]] -name = "unicode-linebreak" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5faade31a542b8b35855fff6e8def199853b2da8da256da52f52f1316ee3137" -dependencies = [ - "hashbrown", - "regex", -] - [[package]] name = "unicode-normalization" version = "0.1.22" diff --git a/Cargo.toml b/Cargo.toml index ab91b3ac04..89f15ffe0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,6 @@ strum = { version = "0.24.1", features = ["strum_macros"] } strum_macros = { version = "0.24.3" } syn = { version = "2.0.15" } test-case = { version = "3.0.0" } -textwrap = { version = "0.16.0" } toml = { version = "0.7.2" } [profile.release] diff --git a/crates/ruff/Cargo.toml b/crates/ruff/Cargo.toml index e13a694c97..b27307b5f9 100644 --- a/crates/ruff/Cargo.toml +++ b/crates/ruff/Cargo.toml @@ -23,6 +23,7 @@ ruff_python_semantic = { path = "../ruff_python_semantic" } ruff_python_stdlib = { path = "../ruff_python_stdlib" } ruff_rustpython = { path = "../ruff_rustpython" } ruff_text_size = { workspace = true } +ruff_textwrap = { path = "../ruff_textwrap" } annotate-snippets = { version = "0.9.1", features = ["color"] } anyhow = { workspace = true } @@ -67,7 +68,6 @@ shellexpand = { workspace = true } smallvec = { workspace = true } strum = { workspace = true } strum_macros = { workspace = true } -textwrap = { workspace = true } thiserror = { version = "1.0.38" } toml = { workspace = true } typed-arena = { version = "2.0.2" } diff --git a/crates/ruff/src/rules/isort/rules/organize_imports.rs b/crates/ruff/src/rules/isort/rules/organize_imports.rs index 7e63d2280f..9b8140d610 100644 --- a/crates/ruff/src/rules/isort/rules/organize_imports.rs +++ b/crates/ruff/src/rules/isort/rules/organize_imports.rs @@ -3,15 +3,16 @@ use std::path::Path; use itertools::{EitherOrBoth, Itertools}; use ruff_text_size::TextRange; use rustpython_parser::ast::{Ranged, Stmt}; -use textwrap::indent; use ruff_diagnostics::{AutofixKind, Diagnostic, Edit, Fix, Violation}; use ruff_macros::{derive_message_formats, violation}; +use ruff_newlines::StrExt; use ruff_python_ast::helpers::{ followed_by_multi_statement_line, preceded_by_multi_statement_line, trailing_lines_end, }; use ruff_python_ast::source_code::{Indexer, Locator, Stylist}; use ruff_python_ast::whitespace::leading_space; +use ruff_textwrap::indent; use crate::line_width::LineWidth; use crate::registry::AsRule; @@ -69,8 +70,8 @@ fn extract_indentation_range(body: &[&Stmt], locator: &Locator) -> TextRange { /// Compares two strings, returning true if they are equal modulo whitespace /// at the start of each line. fn matches_ignoring_indentation(val1: &str, val2: &str) -> bool { - val1.lines() - .zip_longest(val2.lines()) + val1.universal_newlines() + .zip_longest(val2.universal_newlines()) .all(|pair| match pair { EitherOrBoth::Both(line1, line2) => line1.trim_start() == line2.trim_start(), _ => false, @@ -153,7 +154,7 @@ pub(crate) fn organize_imports( let mut diagnostic = Diagnostic::new(UnsortedImports, range); if settings.rules.should_fix(diagnostic.kind.rule()) { diagnostic.set_fix(Fix::automatic(Edit::range_replacement( - indent(&expected, indentation), + indent(&expected, indentation).to_string(), range, ))); } diff --git a/crates/ruff/src/rules/pydocstyle/rules/sections.rs b/crates/ruff/src/rules/pydocstyle/rules/sections.rs index 4412d7713e..458d87ac4b 100644 --- a/crates/ruff/src/rules/pydocstyle/rules/sections.rs +++ b/crates/ruff/src/rules/pydocstyle/rules/sections.rs @@ -13,6 +13,7 @@ use ruff_python_ast::helpers::identifier_range; use ruff_python_ast::{cast, whitespace}; use ruff_python_semantic::analyze::visibility::is_staticmethod; use ruff_python_semantic::definition::{Definition, Member, MemberKind}; +use ruff_textwrap::dedent; use crate::checkers::ast::Checker; use crate::docstrings::sections::{SectionContext, SectionContexts, SectionKind}; @@ -780,7 +781,7 @@ fn args_section(context: &SectionContext) -> FxHashSet { .map(|l| l.as_str()) .filter(|line| line.starts_with(leading_space) || line.is_empty()) .join("\n"); - let args_content = textwrap::dedent(&relevant_lines); + let args_content = dedent(&relevant_lines); // Reformat each section. let mut args_sections: Vec = vec![]; diff --git a/crates/ruff/src/rules/pyflakes/mod.rs b/crates/ruff/src/rules/pyflakes/mod.rs index 7921a91050..b197dd6d62 100644 --- a/crates/ruff/src/rules/pyflakes/mod.rs +++ b/crates/ruff/src/rules/pyflakes/mod.rs @@ -10,9 +10,9 @@ mod tests { use anyhow::Result; use regex::Regex; + use ruff_textwrap::dedent; use rustpython_parser::lexer::LexResult; use test_case::test_case; - use textwrap::dedent; use ruff_diagnostics::Diagnostic; use ruff_python_ast::source_code::{Indexer, Locator, Stylist}; diff --git a/crates/ruff/src/test.rs b/crates/ruff/src/test.rs index df240c2e93..d9f04087eb 100644 --- a/crates/ruff/src/test.rs +++ b/crates/ruff/src/test.rs @@ -1,13 +1,13 @@ #![cfg(test)] +//! Helper functions for the tests of rule implementations. -/// Helper functions for the tests of rule implementations. use std::path::Path; use anyhow::Result; use itertools::Itertools; +use ruff_textwrap::dedent; use rustc_hash::FxHashMap; use rustpython_parser::lexer::LexResult; -use textwrap::dedent; use ruff_diagnostics::{AutofixKind, Diagnostic}; use ruff_python_ast::source_code::{Indexer, Locator, SourceFileBuilder, Stylist}; diff --git a/crates/ruff_cli/Cargo.toml b/crates/ruff_cli/Cargo.toml index ec45b03b61..1f2b3f24f5 100644 --- a/crates/ruff_cli/Cargo.toml +++ b/crates/ruff_cli/Cargo.toml @@ -27,6 +27,7 @@ ruff_cache = { path = "../ruff_cache" } ruff_diagnostics = { path = "../ruff_diagnostics" } ruff_python_ast = { path = "../ruff_python_ast" } ruff_text_size = { workspace = true } +ruff_textwrap = { path = "../ruff_textwrap" } annotate-snippets = { version = "0.9.1", features = ["color"] } anyhow = { workspace = true } @@ -56,7 +57,6 @@ serde_json = { workspace = true } shellexpand = { workspace = true } similar = { workspace = true } strum = { workspace = true, features = [] } -textwrap = { workspace = true } walkdir = { version = "2.3.2" } wild = { version = "2" } diff --git a/crates/ruff_dev/Cargo.toml b/crates/ruff_dev/Cargo.toml index e1d99d296b..4fa0405c9a 100644 --- a/crates/ruff_dev/Cargo.toml +++ b/crates/ruff_dev/Cargo.toml @@ -9,6 +9,7 @@ rust-version = { workspace = true } ruff = { path = "../ruff", features = ["schemars"] } ruff_cli = { path = "../ruff_cli" } ruff_diagnostics = { path = "../ruff_diagnostics" } +ruff_textwrap = { path = "../ruff_textwrap" } anyhow = { workspace = true } clap = { workspace = true } @@ -23,4 +24,3 @@ schemars = { workspace = true } serde_json = { workspace = true } strum = { workspace = true } strum_macros = { workspace = true } -textwrap = { workspace = true } diff --git a/crates/ruff_macros/Cargo.toml b/crates/ruff_macros/Cargo.toml index 4d95a74a2e..f774aee299 100644 --- a/crates/ruff_macros/Cargo.toml +++ b/crates/ruff_macros/Cargo.toml @@ -10,8 +10,9 @@ proc-macro = true doctest = false [dependencies] +ruff_textwrap = { path = "../ruff_textwrap" } + proc-macro2 = { workspace = true } quote = { workspace = true } syn = { workspace = true, features = ["derive", "parsing", "extra-traits", "full"] } -textwrap = { workspace = true } itertools = { workspace = true } diff --git a/crates/ruff_macros/src/config.rs b/crates/ruff_macros/src/config.rs index 11550fde94..38dcaef14b 100644 --- a/crates/ruff_macros/src/config.rs +++ b/crates/ruff_macros/src/config.rs @@ -1,3 +1,5 @@ +use ruff_textwrap::dedent; + use quote::{quote, quote_spanned}; use syn::parse::{Parse, ParseStream}; use syn::spanned::Spanned; @@ -126,7 +128,7 @@ fn handle_option( docs: Vec<&Attribute>, ) -> syn::Result { // Convert the list of `doc` attributes into a single string. - let doc = textwrap::dedent( + let doc = dedent( &docs .into_iter() .map(parse_doc) @@ -179,7 +181,7 @@ impl Parse for FieldAttributes { Ok(Self { default, value_type, - example: textwrap::dedent(&example).trim_matches('\n').to_string(), + example: dedent(&example).trim_matches('\n').to_string(), }) } } diff --git a/crates/ruff_newlines/src/lib.rs b/crates/ruff_newlines/src/lib.rs index 18f9e26a63..919d4ac231 100644 --- a/crates/ruff_newlines/src/lib.rs +++ b/crates/ruff_newlines/src/lib.rs @@ -232,23 +232,29 @@ impl<'a> Line<'a> { TextRange::new(self.start(), self.end()) } + /// Returns the line's new line character, if any. + #[inline] + pub fn line_ending(&self) -> Option { + let mut bytes = self.text.bytes().rev(); + match bytes.next() { + Some(b'\n') => { + if bytes.next() == Some(b'\r') { + Some(LineEnding::CrLf) + } else { + Some(LineEnding::Lf) + } + } + Some(b'\r') => Some(LineEnding::Cr), + _ => None, + } + } + /// Returns the text of the line, excluding the terminating new line character. #[inline] pub fn as_str(&self) -> &'a str { - let mut bytes = self.text.bytes().rev(); - - let newline_len = match bytes.next() { - Some(b'\n') => { - if bytes.next() == Some(b'\r') { - 2 - } else { - 1 - } - } - Some(b'\r') => 1, - _ => 0, - }; - + let newline_len = self + .line_ending() + .map_or(0, |line_ending| line_ending.len()); &self.text[..self.text.len() - newline_len] } diff --git a/crates/ruff_textwrap/Cargo.toml b/crates/ruff_textwrap/Cargo.toml new file mode 100644 index 0000000000..a5217c16cc --- /dev/null +++ b/crates/ruff_textwrap/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "ruff_textwrap" +version = "0.0.0" +publish = false +edition = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +ruff_newlines = { path = "../ruff_newlines" } +ruff_text_size = { workspace = true } diff --git a/crates/ruff_textwrap/src/lib.rs b/crates/ruff_textwrap/src/lib.rs new file mode 100644 index 0000000000..bd803851df --- /dev/null +++ b/crates/ruff_textwrap/src/lib.rs @@ -0,0 +1,336 @@ +//! Functions related to adding and removing indentation from lines of +//! text. + +use std::borrow::Cow; +use std::cmp; + +use ruff_newlines::StrExt; + +/// Indent each line by the given prefix. +/// +/// # Examples +/// +/// ``` +/// use ruff_textwrap::indent; +/// +/// assert_eq!(indent("First line.\nSecond line.\n", " "), +/// " First line.\n Second line.\n"); +/// ``` +/// +/// When indenting, trailing whitespace is stripped from the prefix. +/// This means that empty lines remain empty afterwards: +/// +/// ``` +/// use ruff_textwrap::indent; +/// +/// assert_eq!(indent("First line.\n\n\nSecond line.\n", " "), +/// " First line.\n\n\n Second line.\n"); +/// ``` +/// +/// Notice how `"\n\n\n"` remained as `"\n\n\n"`. +/// +/// This feature is useful when you want to indent text and have a +/// space between your prefix and the text. In this case, you _don't_ +/// want a trailing space on empty lines: +/// +/// ``` +/// use ruff_textwrap::indent; +/// +/// assert_eq!(indent("foo = 123\n\nprint(foo)\n", "# "), +/// "# foo = 123\n#\n# print(foo)\n"); +/// ``` +/// +/// Notice how `"\n\n"` became `"\n#\n"` instead of `"\n# \n"` which +/// would have trailing whitespace. +/// +/// Leading and trailing whitespace coming from the text itself is +/// kept unchanged: +/// +/// ``` +/// use ruff_textwrap::indent; +/// +/// assert_eq!(indent(" \t Foo ", "->"), "-> \t Foo "); +/// ``` +pub fn indent<'a>(text: &'a str, prefix: &str) -> Cow<'a, str> { + if prefix.is_empty() { + return Cow::Borrowed(text); + } + + let mut result = String::with_capacity(text.len() + prefix.len()); + let trimmed_prefix = prefix.trim_end(); + for line in text.universal_newlines() { + if line.trim().is_empty() { + result.push_str(trimmed_prefix); + } else { + result.push_str(prefix); + } + result.push_str(line.as_full_str()); + } + Cow::Owned(result) +} + +/// Removes common leading whitespace from each line. +/// +/// This function will look at each non-empty line and determine the +/// maximum amount of whitespace that can be removed from all lines: +/// +/// ``` +/// use ruff_textwrap::dedent; +/// +/// assert_eq!(dedent(" +/// 1st line +/// 2nd line +/// 3rd line +/// "), " +/// 1st line +/// 2nd line +/// 3rd line +/// "); +/// ``` +pub fn dedent(text: &str) -> Cow<'_, str> { + // Find the minimum amount of leading whitespace on each line. + let prefix_len = text + .universal_newlines() + .fold(usize::MAX, |prefix_len, line| { + let leading_whitespace_len = line.len() - line.trim_start().len(); + if leading_whitespace_len == line.len() { + // Skip empty lines. + prefix_len + } else { + cmp::min(prefix_len, leading_whitespace_len) + } + }); + + // If there is no common prefix, no need to dedent. + if prefix_len == usize::MAX { + return Cow::Borrowed(text); + } + + // Remove the common prefix from each line. + let mut result = String::with_capacity(text.len()); + for line in text.universal_newlines() { + if line.trim().is_empty() { + if let Some(line_ending) = line.line_ending() { + result.push_str(&line_ending); + } + } else { + result.push_str(&line.as_full_str()[prefix_len..]); + } + } + Cow::Owned(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn indent_empty() { + assert_eq!(indent("\n", " "), "\n"); + } + + #[test] + #[rustfmt::skip] + fn indent_nonempty() { + let text = [ + " foo\n", + "bar\n", + " baz\n", + ].join(""); + let expected = [ + "// foo\n", + "// bar\n", + "// baz\n", + ].join(""); + assert_eq!(indent(&text, "// "), expected); + } + + #[test] + #[rustfmt::skip] + fn indent_empty_line() { + let text = [ + " foo", + "bar", + "", + " baz", + ].join("\n"); + let expected = [ + "// foo", + "// bar", + "//", + "// baz", + ].join("\n"); + assert_eq!(indent(&text, "// "), expected); + } + + #[test] + #[rustfmt::skip] + fn indent_mixed_newlines() { + let text = [ + " foo\r\n", + "bar\n", + " baz\r", + ].join(""); + let expected = [ + "// foo\r\n", + "// bar\n", + "// baz\r", + ].join(""); + assert_eq!(indent(&text, "// "), expected); + } + + #[test] + fn dedent_empty() { + assert_eq!(dedent(""), ""); + } + + #[test] + #[rustfmt::skip] + fn dedent_multi_line() { + let x = [ + " foo", + " bar", + " baz", + ].join("\n"); + let y = [ + " foo", + "bar", + " baz" + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_empty_line() { + let x = [ + " foo", + " bar", + " ", + " baz" + ].join("\n"); + let y = [ + " foo", + "bar", + "", + " baz" + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_blank_line() { + let x = [ + " foo", + "", + " bar", + " foo", + " bar", + " baz", + ].join("\n"); + let y = [ + "foo", + "", + " bar", + " foo", + " bar", + " baz", + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_whitespace_line() { + let x = [ + " foo", + " ", + " bar", + " foo", + " bar", + " baz", + ].join("\n"); + let y = [ + "foo", + "", + " bar", + " foo", + " bar", + " baz", + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_mixed_whitespace() { + let x = [ + "\tfoo", + " bar", + ].join("\n"); + let y = [ + "foo", + " bar", + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_tabbed_whitespace() { + let x = [ + "\t\tfoo", + "\t\t\tbar", + ].join("\n"); + let y = [ + "foo", + "\tbar", + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_mixed_tabbed_whitespace() { + let x = [ + "\t \tfoo", + "\t \t\tbar", + ].join("\n"); + let y = [ + "foo", + "\tbar", + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_preserve_no_terminating_newline() { + let x = [ + " foo", + " bar", + ].join("\n"); + let y = [ + "foo", + " bar", + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_mixed_newlines() { + let x = [ + " foo\r\n", + " bar\n", + " baz\r", + ].join(""); + let y = [ + " foo\r\n", + "bar\n", + " baz\r" + ].join(""); + assert_eq!(dedent(&x), y); + } +}