From 27912d46b1af2db0d290b3598e46aca04a35127f Mon Sep 17 00:00:00 2001 From: Denys Zhak Date: Thu, 11 Dec 2025 13:04:57 +0100 Subject: [PATCH] Remove `BackwardsTokenizer` based `parenthesized_range` references in `ruff_linter` (#21836) Co-authored-by: Micha Reiser --- crates/ruff_linter/src/checkers/ast/mod.rs | 9 ++++ crates/ruff_linter/src/fix/edits.rs | 28 ++++------- .../rules/fastapi_redundant_response_model.rs | 4 +- .../rules/map_without_explicit_strict.rs | 7 +-- .../rules/mutable_argument_default.rs | 18 ++----- .../rules/no_explicit_stacklevel.rs | 7 +-- .../rules/zip_without_explicit_strict.rs | 7 +-- .../rules/unnecessary_generator_list.rs | 12 ++--- .../rules/unnecessary_generator_set.rs | 12 ++--- .../unnecessary_list_comprehension_set.rs | 12 ++--- .../rules/explicit.rs | 10 +--- .../rules/exc_info_outside_except_handler.rs | 5 +- .../rules/unnecessary_dict_kwargs.rs | 9 ++-- .../rules/unnecessary_range_start.rs | 6 +-- .../rules/generic_not_last_base_class.rs | 10 ++-- .../rules/redundant_none_literal.rs | 30 ++++-------- .../flake8_pytest_style/rules/assertion.rs | 5 +- .../flake8_pytest_style/rules/fixture.rs | 4 +- .../flake8_pytest_style/rules/parametrize.rs | 35 +++----------- .../flake8_simplify/rules/ast_bool_op.rs | 24 +++------- .../rules/flake8_simplify/rules/ast_ifexp.rs | 11 ++--- .../rules/if_with_same_arms.rs | 48 ++++++++----------- .../flake8_simplify/rules/key_in_dict.rs | 20 ++------ .../rules/type_alias_quotes.rs | 13 +++-- .../path_constructor_current_directory.rs | 16 ++----- .../pandas_vet/rules/inplace_argument.rs | 20 +++----- .../pycodestyle/rules/lambda_assignment.rs | 30 ++++-------- .../pycodestyle/rules/literal_comparisons.rs | 24 ++++------ .../src/rules/pycodestyle/rules/not_tests.rs | 4 +- .../src/rules/pyflakes/rules/repeated_keys.rs | 14 ++---- .../rules/pyflakes/rules/unused_variable.rs | 14 ++---- .../rules/boolean_chained_comparison.rs | 42 +++++----------- .../src/rules/pylint/rules/duplicate_bases.rs | 2 +- .../src/rules/pylint/rules/if_stmt_min_max.rs | 11 ++--- .../pylint/rules/missing_maxsplit_arg.rs | 8 +--- .../pylint/rules/non_augmented_assignment.rs | 8 ++-- .../rules/subprocess_run_without_check.rs | 7 +-- .../pylint/rules/unspecified_encoding.rs | 6 +-- .../rules/pep695/non_pep695_generic_class.rs | 2 +- .../rules/pep695/non_pep695_type_alias.rs | 6 +-- .../pyupgrade/rules/replace_stdout_stderr.rs | 9 ++-- .../rules/replace_universal_newlines.rs | 2 +- .../rules/unnecessary_encode_utf8.rs | 8 ++-- .../rules/useless_class_metaclass_type.rs | 2 +- .../rules/useless_object_inheritance.rs | 2 +- .../pyupgrade/rules/yield_in_for_loop.rs | 11 ++--- .../ruff_linter/src/rules/refurb/helpers.rs | 10 ++-- .../refurb/rules/fromisoformat_replace_z.rs | 5 +- .../rules/if_exp_instead_of_or_operator.rs | 16 ++----- .../rules/refurb/rules/readlines_in_for.rs | 5 +- .../rules/refurb/rules/redundant_log_base.rs | 11 ++--- .../rules/single_item_membership_test.rs | 2 +- .../ruff/rules/class_with_mixed_type_vars.rs | 2 +- .../rules/ruff/rules/default_factory_kwarg.rs | 11 ++--- .../ruff/rules/falsy_dict_get_fallback.rs | 2 +- .../rules/parenthesize_chained_operators.rs | 11 +---- .../src/rules/ruff/rules/post_init_default.rs | 5 +- .../ruff/rules/quadratic_list_summation.rs | 11 ++--- .../src/rules/ruff/rules/starmap_zip.rs | 20 ++------ .../ruff/rules/unnecessary_cast_to_int.rs | 33 ++++++------- .../rules/ruff/rules/unnecessary_key_check.rs | 20 ++------ .../unnecessary_literal_within_deque_call.rs | 22 ++------- crates/ruff_python_ast/src/helpers.rs | 12 ++--- 63 files changed, 263 insertions(+), 529 deletions(-) diff --git a/crates/ruff_linter/src/checkers/ast/mod.rs b/crates/ruff_linter/src/checkers/ast/mod.rs index 4d4d7e9293..37422cbf18 100644 --- a/crates/ruff_linter/src/checkers/ast/mod.rs +++ b/crates/ruff_linter/src/checkers/ast/mod.rs @@ -437,6 +437,15 @@ impl<'a> Checker<'a> { } } + /// Returns the [`Tokens`] for the parsed source file. + /// + /// + /// Unlike [`Self::tokens`], this method always returns + /// the tokens for the current file, even when within a parsed type annotation. + pub(crate) fn source_tokens(&self) -> &'a Tokens { + self.parsed.tokens() + } + /// The [`Locator`] for the current file, which enables extraction of source code from byte /// offsets. pub(crate) const fn locator(&self) -> &'a Locator<'a> { diff --git a/crates/ruff_linter/src/fix/edits.rs b/crates/ruff_linter/src/fix/edits.rs index 05e90519eb..20f50d6e10 100644 --- a/crates/ruff_linter/src/fix/edits.rs +++ b/crates/ruff_linter/src/fix/edits.rs @@ -3,14 +3,13 @@ use anyhow::{Context, Result}; use ruff_python_ast::AnyNodeRef; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::{self, Tokens, parenthesized_range}; use ruff_python_ast::{self as ast, Arguments, ExceptHandler, Expr, ExprList, Parameters, Stmt}; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; use ruff_python_trivia::textwrap::dedent_to; use ruff_python_trivia::{ - CommentRanges, PythonWhitespace, SimpleTokenKind, SimpleTokenizer, has_leading_content, - is_python_whitespace, + PythonWhitespace, SimpleTokenKind, SimpleTokenizer, has_leading_content, is_python_whitespace, }; use ruff_source_file::{LineRanges, NewlineWithTrailingNewline, UniversalNewlines}; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; @@ -209,7 +208,7 @@ pub(crate) fn remove_argument( arguments: &Arguments, parentheses: Parentheses, source: &str, - comment_ranges: &CommentRanges, + tokens: &Tokens, ) -> Result { // Partition into arguments before and after the argument to remove. let (before, after): (Vec<_>, Vec<_>) = arguments @@ -224,7 +223,7 @@ pub(crate) fn remove_argument( .context("Unable to find argument")?; let parenthesized_range = - parenthesized_range(arg.value().into(), arguments.into(), comment_ranges, source) + token::parenthesized_range(arg.value().into(), arguments.into(), tokens) .unwrap_or(arg.range()); if !after.is_empty() { @@ -270,25 +269,14 @@ pub(crate) fn remove_argument( /// /// The new argument will be inserted before the first existing keyword argument in `arguments`, if /// there are any present. Otherwise, the new argument is added to the end of the argument list. -pub(crate) fn add_argument( - argument: &str, - arguments: &Arguments, - comment_ranges: &CommentRanges, - source: &str, -) -> Edit { +pub(crate) fn add_argument(argument: &str, arguments: &Arguments, tokens: &Tokens) -> Edit { if let Some(ast::Keyword { range, value, .. }) = arguments.keywords.first() { - let keyword = parenthesized_range(value.into(), arguments.into(), comment_ranges, source) - .unwrap_or(*range); + let keyword = parenthesized_range(value.into(), arguments.into(), tokens).unwrap_or(*range); Edit::insertion(format!("{argument}, "), keyword.start()) } else if let Some(last) = arguments.arguments_source_order().last() { // Case 1: existing arguments, so append after the last argument. - let last = parenthesized_range( - last.value().into(), - arguments.into(), - comment_ranges, - source, - ) - .unwrap_or(last.range()); + let last = parenthesized_range(last.value().into(), arguments.into(), tokens) + .unwrap_or(last.range()); Edit::insertion(format!(", {argument}"), last.end()) } else { // Case 2: no arguments. Add argument, without any trailing comma. diff --git a/crates/ruff_linter/src/rules/fastapi/rules/fastapi_redundant_response_model.rs b/crates/ruff_linter/src/rules/fastapi/rules/fastapi_redundant_response_model.rs index 440be901a7..d31ffbf61e 100644 --- a/crates/ruff_linter/src/rules/fastapi/rules/fastapi_redundant_response_model.rs +++ b/crates/ruff_linter/src/rules/fastapi/rules/fastapi_redundant_response_model.rs @@ -91,8 +91,8 @@ pub(crate) fn fastapi_redundant_response_model(checker: &Checker, function_def: response_model_arg, &call.arguments, Parentheses::Preserve, - checker.locator().contents(), - checker.comment_ranges(), + checker.source(), + checker.tokens(), ) .map(Fix::unsafe_edit) }); diff --git a/crates/ruff_linter/src/rules/flake8_bugbear/rules/map_without_explicit_strict.rs b/crates/ruff_linter/src/rules/flake8_bugbear/rules/map_without_explicit_strict.rs index cd268f610b..04e9640e09 100644 --- a/crates/ruff_linter/src/rules/flake8_bugbear/rules/map_without_explicit_strict.rs +++ b/crates/ruff_linter/src/rules/flake8_bugbear/rules/map_without_explicit_strict.rs @@ -74,12 +74,7 @@ pub(crate) fn map_without_explicit_strict(checker: &Checker, call: &ast::ExprCal checker .report_diagnostic(MapWithoutExplicitStrict, call.range()) .set_fix(Fix::applicable_edit( - add_argument( - "strict=False", - &call.arguments, - checker.comment_ranges(), - checker.locator().contents(), - ), + add_argument("strict=False", &call.arguments, checker.tokens()), Applicability::Unsafe, )); } diff --git a/crates/ruff_linter/src/rules/flake8_bugbear/rules/mutable_argument_default.rs b/crates/ruff_linter/src/rules/flake8_bugbear/rules/mutable_argument_default.rs index 94098831b8..c0b1b5e840 100644 --- a/crates/ruff_linter/src/rules/flake8_bugbear/rules/mutable_argument_default.rs +++ b/crates/ruff_linter/src/rules/flake8_bugbear/rules/mutable_argument_default.rs @@ -3,7 +3,7 @@ use std::fmt::Write; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::helpers::is_docstring_stmt; use ruff_python_ast::name::QualifiedName; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{self as ast, Expr, ParameterWithDefault}; use ruff_python_semantic::SemanticModel; use ruff_python_semantic::analyze::function_type::is_stub; @@ -166,12 +166,7 @@ fn move_initialization( return None; } - let range = match parenthesized_range( - default.into(), - parameter.into(), - checker.comment_ranges(), - checker.source(), - ) { + let range = match parenthesized_range(default.into(), parameter.into(), checker.tokens()) { Some(range) => range, None => default.range(), }; @@ -194,13 +189,8 @@ fn move_initialization( "{} = {}", parameter.parameter.name(), locator.slice( - parenthesized_range( - default.into(), - parameter.into(), - checker.comment_ranges(), - checker.source() - ) - .unwrap_or(default.range()) + parenthesized_range(default.into(), parameter.into(), checker.tokens()) + .unwrap_or(default.range()) ) ); } else { diff --git a/crates/ruff_linter/src/rules/flake8_bugbear/rules/no_explicit_stacklevel.rs b/crates/ruff_linter/src/rules/flake8_bugbear/rules/no_explicit_stacklevel.rs index f737e781ef..a21cf9a30e 100644 --- a/crates/ruff_linter/src/rules/flake8_bugbear/rules/no_explicit_stacklevel.rs +++ b/crates/ruff_linter/src/rules/flake8_bugbear/rules/no_explicit_stacklevel.rs @@ -92,12 +92,7 @@ pub(crate) fn no_explicit_stacklevel(checker: &Checker, call: &ast::ExprCall) { } let mut diagnostic = checker.report_diagnostic(NoExplicitStacklevel, call.func.range()); - let edit = add_argument( - "stacklevel=2", - &call.arguments, - checker.comment_ranges(), - checker.locator().contents(), - ); + let edit = add_argument("stacklevel=2", &call.arguments, checker.tokens()); diagnostic.set_fix(Fix::unsafe_edit(edit)); } diff --git a/crates/ruff_linter/src/rules/flake8_bugbear/rules/zip_without_explicit_strict.rs b/crates/ruff_linter/src/rules/flake8_bugbear/rules/zip_without_explicit_strict.rs index 136715b981..db71c7b2fb 100644 --- a/crates/ruff_linter/src/rules/flake8_bugbear/rules/zip_without_explicit_strict.rs +++ b/crates/ruff_linter/src/rules/flake8_bugbear/rules/zip_without_explicit_strict.rs @@ -70,12 +70,7 @@ pub(crate) fn zip_without_explicit_strict(checker: &Checker, call: &ast::ExprCal checker .report_diagnostic(ZipWithoutExplicitStrict, call.range()) .set_fix(Fix::applicable_edit( - add_argument( - "strict=False", - &call.arguments, - checker.comment_ranges(), - checker.locator().contents(), - ), + add_argument("strict=False", &call.arguments, checker.tokens()), Applicability::Unsafe, )); } diff --git a/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_list.rs b/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_list.rs index 5fdc1a37a3..d271a13792 100644 --- a/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_list.rs +++ b/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_list.rs @@ -2,8 +2,8 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast as ast; use ruff_python_ast::ExprGenerator; use ruff_python_ast::comparable::ComparableExpr; -use ruff_python_ast::parenthesize::parenthesized_range; use ruff_python_ast::token::TokenKind; +use ruff_python_ast::token::parenthesized_range; use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::checkers::ast::Checker; @@ -142,13 +142,9 @@ pub(crate) fn unnecessary_generator_list(checker: &Checker, call: &ast::ExprCall if *parenthesized { // The generator's range will include the innermost parentheses, but it could be // surrounded by additional parentheses. - let range = parenthesized_range( - argument.into(), - (&call.arguments).into(), - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(argument.range()); + let range = + parenthesized_range(argument.into(), (&call.arguments).into(), checker.tokens()) + .unwrap_or(argument.range()); // The generator always parenthesizes the expression; trim the parentheses. let generator = checker.generator().expr(argument); diff --git a/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_set.rs b/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_set.rs index 0560935bae..05a1c523cf 100644 --- a/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_set.rs +++ b/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_set.rs @@ -2,8 +2,8 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast as ast; use ruff_python_ast::ExprGenerator; use ruff_python_ast::comparable::ComparableExpr; -use ruff_python_ast::parenthesize::parenthesized_range; use ruff_python_ast::token::TokenKind; +use ruff_python_ast::token::parenthesized_range; use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::checkers::ast::Checker; @@ -147,13 +147,9 @@ pub(crate) fn unnecessary_generator_set(checker: &Checker, call: &ast::ExprCall) if *parenthesized { // The generator's range will include the innermost parentheses, but it could be // surrounded by additional parentheses. - let range = parenthesized_range( - argument.into(), - (&call.arguments).into(), - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(argument.range()); + let range = + parenthesized_range(argument.into(), (&call.arguments).into(), checker.tokens()) + .unwrap_or(argument.range()); // The generator always parenthesizes the expression; trim the parentheses. let generator = checker.generator().expr(argument); diff --git a/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_list_comprehension_set.rs b/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_list_comprehension_set.rs index b4fda738e2..f6699500af 100644 --- a/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_list_comprehension_set.rs +++ b/crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_list_comprehension_set.rs @@ -1,7 +1,7 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast as ast; -use ruff_python_ast::parenthesize::parenthesized_range; use ruff_python_ast::token::TokenKind; +use ruff_python_ast::token::parenthesized_range; use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::checkers::ast::Checker; @@ -89,13 +89,9 @@ pub(crate) fn unnecessary_list_comprehension_set(checker: &Checker, call: &ast:: // If the list comprehension is parenthesized, remove the parentheses in addition to // removing the brackets. - let replacement_range = parenthesized_range( - argument.into(), - (&call.arguments).into(), - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or_else(|| argument.range()); + let replacement_range = + parenthesized_range(argument.into(), (&call.arguments).into(), checker.tokens()) + .unwrap_or_else(|| argument.range()); let span = argument.range().add_start(one).sub_end(one); let replacement = diff --git a/crates/ruff_linter/src/rules/flake8_implicit_str_concat/rules/explicit.rs b/crates/ruff_linter/src/rules/flake8_implicit_str_concat/rules/explicit.rs index 8edb7b46ad..e742b8922d 100644 --- a/crates/ruff_linter/src/rules/flake8_implicit_str_concat/rules/explicit.rs +++ b/crates/ruff_linter/src/rules/flake8_implicit_str_concat/rules/explicit.rs @@ -1,5 +1,5 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{self as ast, Expr, Operator}; use ruff_python_trivia::is_python_whitespace; use ruff_source_file::LineRanges; @@ -88,13 +88,7 @@ pub(crate) fn explicit(checker: &Checker, expr: &Expr) { checker.report_diagnostic(ExplicitStringConcatenation, expr.range()); let is_parenthesized = |expr: &Expr| { - parenthesized_range( - expr.into(), - bin_op.into(), - checker.comment_ranges(), - checker.source(), - ) - .is_some() + parenthesized_range(expr.into(), bin_op.into(), checker.tokens()).is_some() }; // If either `left` or `right` is parenthesized, generating // a fix would be too involved. Just report the diagnostic. diff --git a/crates/ruff_linter/src/rules/flake8_logging/rules/exc_info_outside_except_handler.rs b/crates/ruff_linter/src/rules/flake8_logging/rules/exc_info_outside_except_handler.rs index 1172e40893..7d18897708 100644 --- a/crates/ruff_linter/src/rules/flake8_logging/rules/exc_info_outside_except_handler.rs +++ b/crates/ruff_linter/src/rules/flake8_logging/rules/exc_info_outside_except_handler.rs @@ -111,7 +111,6 @@ pub(crate) fn exc_info_outside_except_handler(checker: &Checker, call: &ExprCall } let arguments = &call.arguments; - let source = checker.source(); let mut diagnostic = checker.report_diagnostic(ExcInfoOutsideExceptHandler, exc_info.range); @@ -120,8 +119,8 @@ pub(crate) fn exc_info_outside_except_handler(checker: &Checker, call: &ExprCall exc_info, arguments, Parentheses::Preserve, - source, - checker.comment_ranges(), + checker.source(), + checker.tokens(), )?; Ok(Fix::unsafe_edit(edit)) }); diff --git a/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_dict_kwargs.rs b/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_dict_kwargs.rs index cc436d258a..1b3faba504 100644 --- a/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_dict_kwargs.rs +++ b/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_dict_kwargs.rs @@ -2,7 +2,7 @@ use itertools::Itertools; use rustc_hash::{FxBuildHasher, FxHashSet}; use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{self as ast, Expr}; use ruff_python_stdlib::identifiers::is_identifier; use ruff_text_size::Ranged; @@ -129,8 +129,8 @@ pub(crate) fn unnecessary_dict_kwargs(checker: &Checker, call: &ast::ExprCall) { keyword, &call.arguments, Parentheses::Preserve, - checker.locator().contents(), - checker.comment_ranges(), + checker.source(), + checker.tokens(), ) .map(Fix::safe_edit) }); @@ -158,8 +158,7 @@ pub(crate) fn unnecessary_dict_kwargs(checker: &Checker, call: &ast::ExprCall) { parenthesized_range( value.into(), dict.into(), - checker.comment_ranges(), - checker.locator().contents(), + checker.tokens() ) .unwrap_or(value.range()) ) diff --git a/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_range_start.rs b/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_range_start.rs index 6e6b10b206..e12af6e069 100644 --- a/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_range_start.rs +++ b/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_range_start.rs @@ -73,11 +73,11 @@ pub(crate) fn unnecessary_range_start(checker: &Checker, call: &ast::ExprCall) { let mut diagnostic = checker.report_diagnostic(UnnecessaryRangeStart, start.range()); diagnostic.try_set_fix(|| { remove_argument( - &start, + start, &call.arguments, Parentheses::Preserve, - checker.locator().contents(), - checker.comment_ranges(), + checker.source(), + checker.tokens(), ) .map(Fix::safe_edit) }); diff --git a/crates/ruff_linter/src/rules/flake8_pyi/rules/generic_not_last_base_class.rs b/crates/ruff_linter/src/rules/flake8_pyi/rules/generic_not_last_base_class.rs index 4cd5035693..90b8476809 100644 --- a/crates/ruff_linter/src/rules/flake8_pyi/rules/generic_not_last_base_class.rs +++ b/crates/ruff_linter/src/rules/flake8_pyi/rules/generic_not_last_base_class.rs @@ -160,20 +160,16 @@ fn generate_fix( ) -> anyhow::Result { let locator = checker.locator(); let source = locator.contents(); + let tokens = checker.tokens(); let deletion = remove_argument( generic_base, arguments, Parentheses::Preserve, source, - checker.comment_ranges(), + tokens, )?; - let insertion = add_argument( - locator.slice(generic_base), - arguments, - checker.comment_ranges(), - source, - ); + let insertion = add_argument(locator.slice(generic_base), arguments, tokens); Ok(Fix::unsafe_edits(deletion, [insertion])) } diff --git a/crates/ruff_linter/src/rules/flake8_pyi/rules/redundant_none_literal.rs b/crates/ruff_linter/src/rules/flake8_pyi/rules/redundant_none_literal.rs index 14145229fc..b3e35c21c2 100644 --- a/crates/ruff_linter/src/rules/flake8_pyi/rules/redundant_none_literal.rs +++ b/crates/ruff_linter/src/rules/flake8_pyi/rules/redundant_none_literal.rs @@ -5,7 +5,7 @@ use ruff_python_ast::{ helpers::{pep_604_union, typing_optional}, name::Name, operator_precedence::OperatorPrecedence, - parenthesize::parenthesized_range, + token::{Tokens, parenthesized_range}, }; use ruff_python_semantic::analyze::typing::{traverse_literal, traverse_union}; use ruff_text_size::{Ranged, TextRange}; @@ -243,16 +243,12 @@ fn create_fix( let union_expr = pep_604_union(&[new_literal_expr, none_expr]); // Check if we need parentheses to preserve operator precedence - let content = if needs_parentheses_for_precedence( - semantic, - literal_expr, - checker.comment_ranges(), - checker.source(), - ) { - format!("({})", checker.generator().expr(&union_expr)) - } else { - checker.generator().expr(&union_expr) - }; + let content = + if needs_parentheses_for_precedence(semantic, literal_expr, checker.tokens()) { + format!("({})", checker.generator().expr(&union_expr)) + } else { + checker.generator().expr(&union_expr) + }; let union_edit = Edit::range_replacement(content, literal_expr.range()); Fix::applicable_edit(union_edit, applicability) @@ -278,8 +274,7 @@ enum UnionKind { fn needs_parentheses_for_precedence( semantic: &ruff_python_semantic::SemanticModel, literal_expr: &Expr, - comment_ranges: &ruff_python_trivia::CommentRanges, - source: &str, + tokens: &Tokens, ) -> bool { // Get the parent expression to check if we're in a context that needs parentheses let Some(parent_expr) = semantic.current_expression_parent() else { @@ -287,14 +282,7 @@ fn needs_parentheses_for_precedence( }; // Check if the literal expression is already parenthesized - if parenthesized_range( - literal_expr.into(), - parent_expr.into(), - comment_ranges, - source, - ) - .is_some() - { + if parenthesized_range(literal_expr.into(), parent_expr.into(), tokens).is_some() { return false; // Already parenthesized, don't add more } diff --git a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/assertion.rs b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/assertion.rs index 545372dd6c..c97efd8b05 100644 --- a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/assertion.rs +++ b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/assertion.rs @@ -10,7 +10,7 @@ use libcst_native::{ use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::helpers::Truthiness; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::visitor::Visitor; use ruff_python_ast::{ self as ast, AnyNodeRef, Arguments, BoolOp, ExceptHandler, Expr, Keyword, Stmt, UnaryOp, @@ -303,8 +303,7 @@ pub(crate) fn unittest_assertion( parenthesized_range( expr.into(), checker.semantic().current_statement().into(), - checker.comment_ranges(), - checker.locator().contents(), + checker.tokens(), ) .unwrap_or(expr.range()), ))); diff --git a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/fixture.rs b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/fixture.rs index 49be564b48..c939d44346 100644 --- a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/fixture.rs +++ b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/fixture.rs @@ -768,8 +768,8 @@ fn check_fixture_decorator(checker: &Checker, func_name: &str, decorator: &Decor keyword, arguments, edits::Parentheses::Preserve, - checker.locator().contents(), - checker.comment_ranges(), + checker.source(), + checker.tokens(), ) .map(Fix::unsafe_edit) }); diff --git a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs index 20b14399f9..904ca1c494 100644 --- a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs +++ b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs @@ -2,10 +2,9 @@ use rustc_hash::{FxBuildHasher, FxHashMap}; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::comparable::ComparableExpr; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::{Tokens, parenthesized_range}; use ruff_python_ast::{self as ast, Expr, ExprCall, ExprContext, StringLiteralFlags}; use ruff_python_codegen::Generator; -use ruff_python_trivia::CommentRanges; use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange, TextSize}; @@ -322,18 +321,8 @@ fn elts_to_csv(elts: &[Expr], generator: Generator, flags: StringLiteralFlags) - /// ``` /// /// This method assumes that the first argument is a string. -fn get_parametrize_name_range( - call: &ExprCall, - expr: &Expr, - comment_ranges: &CommentRanges, - source: &str, -) -> Option { - parenthesized_range( - expr.into(), - (&call.arguments).into(), - comment_ranges, - source, - ) +fn get_parametrize_name_range(call: &ExprCall, expr: &Expr, tokens: &Tokens) -> Option { + parenthesized_range(expr.into(), (&call.arguments).into(), tokens) } /// PT006 @@ -349,13 +338,8 @@ fn check_names(checker: &Checker, call: &ExprCall, expr: &Expr, argvalues: &Expr if names.len() > 1 { match names_type { types::ParametrizeNameType::Tuple => { - let name_range = get_parametrize_name_range( - call, - expr, - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(expr.range()); + let name_range = get_parametrize_name_range(call, expr, checker.tokens()) + .unwrap_or(expr.range()); let mut diagnostic = checker.report_diagnostic( PytestParametrizeNamesWrongType { single_argument: false, @@ -386,13 +370,8 @@ fn check_names(checker: &Checker, call: &ExprCall, expr: &Expr, argvalues: &Expr ))); } types::ParametrizeNameType::List => { - let name_range = get_parametrize_name_range( - call, - expr, - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(expr.range()); + let name_range = get_parametrize_name_range(call, expr, checker.tokens()) + .unwrap_or(expr.range()); let mut diagnostic = checker.report_diagnostic( PytestParametrizeNamesWrongType { single_argument: false, diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs index 0b53a271f4..5de313a600 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_bool_op.rs @@ -10,7 +10,7 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::comparable::ComparableExpr; use ruff_python_ast::helpers::{Truthiness, contains_effect}; use ruff_python_ast::name::Name; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_codegen::Generator; use ruff_python_semantic::SemanticModel; @@ -800,14 +800,9 @@ fn is_short_circuit( edit = Some(get_short_circuit_edit( value, TextRange::new( - parenthesized_range( - furthest.into(), - expr.into(), - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(furthest.range()) - .start(), + parenthesized_range(furthest.into(), expr.into(), checker.tokens()) + .unwrap_or(furthest.range()) + .start(), expr.end(), ), short_circuit_truthiness, @@ -828,14 +823,9 @@ fn is_short_circuit( edit = Some(get_short_circuit_edit( next_value, TextRange::new( - parenthesized_range( - furthest.into(), - expr.into(), - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(furthest.range()) - .start(), + parenthesized_range(furthest.into(), expr.into(), checker.tokens()) + .unwrap_or(furthest.range()) + .start(), expr.end(), ), short_circuit_truthiness, diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_ifexp.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_ifexp.rs index 15b2f54bf8..2292f4e581 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_ifexp.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_ifexp.rs @@ -4,7 +4,7 @@ use ruff_text_size::{Ranged, TextRange}; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::helpers::{is_const_false, is_const_true}; use ruff_python_ast::name::Name; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use crate::checkers::ast::Checker; use crate::{AlwaysFixableViolation, Edit, Fix, FixAvailability, Violation}; @@ -171,13 +171,8 @@ pub(crate) fn if_expr_with_true_false( checker .locator() .slice( - parenthesized_range( - test.into(), - expr.into(), - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(test.range()), + parenthesized_range(test.into(), expr.into(), checker.tokens()) + .unwrap_or(test.range()), ) .to_string(), expr.range(), diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/if_with_same_arms.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/if_with_same_arms.rs index 92182b383a..17b7c5c612 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/if_with_same_arms.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/if_with_same_arms.rs @@ -4,10 +4,10 @@ use anyhow::Result; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::comparable::ComparableStmt; -use ruff_python_ast::parenthesize::parenthesized_range; use ruff_python_ast::stmt_if::{IfElifBranch, if_elif_branches}; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{self as ast, Expr}; -use ruff_python_trivia::{CommentRanges, SimpleTokenKind, SimpleTokenizer}; +use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_source_file::LineRanges; use ruff_text_size::{Ranged, TextRange}; @@ -99,7 +99,7 @@ pub(crate) fn if_with_same_arms(checker: &Checker, stmt_if: &ast::StmtIf) { ¤t_branch, following_branch, checker.locator(), - checker.comment_ranges(), + checker.tokens(), ) }); } @@ -111,7 +111,7 @@ fn merge_branches( current_branch: &IfElifBranch, following_branch: &IfElifBranch, locator: &Locator, - comment_ranges: &CommentRanges, + tokens: &ruff_python_ast::token::Tokens, ) -> Result { // Identify the colon (`:`) at the end of the current branch's test. let Some(current_branch_colon) = @@ -127,12 +127,9 @@ fn merge_branches( ); // If the following test isn't parenthesized, consider parenthesizing it. - let following_branch_test = if let Some(range) = parenthesized_range( - following_branch.test.into(), - stmt_if.into(), - comment_ranges, - locator.contents(), - ) { + let following_branch_test = if let Some(range) = + parenthesized_range(following_branch.test.into(), stmt_if.into(), tokens) + { Cow::Borrowed(locator.slice(range)) } else if matches!( following_branch.test, @@ -153,24 +150,19 @@ fn merge_branches( // // For example, if the current test is `x if x else y`, we should parenthesize it to // `(x if x else y) or ...`. - let parenthesize_edit = if matches!( - current_branch.test, - Expr::Lambda(_) | Expr::Named(_) | Expr::If(_) - ) && parenthesized_range( - current_branch.test.into(), - stmt_if.into(), - comment_ranges, - locator.contents(), - ) - .is_none() - { - Some(Edit::range_replacement( - format!("({})", locator.slice(current_branch.test)), - current_branch.test.range(), - )) - } else { - None - }; + let parenthesize_edit = + if matches!( + current_branch.test, + Expr::Lambda(_) | Expr::Named(_) | Expr::If(_) + ) && parenthesized_range(current_branch.test.into(), stmt_if.into(), tokens).is_none() + { + Some(Edit::range_replacement( + format!("({})", locator.slice(current_branch.test)), + current_branch.test.range(), + )) + } else { + None + }; Ok(Fix::safe_edits( deletion_edit, diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/key_in_dict.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/key_in_dict.rs index 5ced08a673..645a79d9e1 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/key_in_dict.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/key_in_dict.rs @@ -1,6 +1,6 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::AnyNodeRef; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{self as ast, Arguments, CmpOp, Comprehension, Expr}; use ruff_python_semantic::analyze::typing; use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; @@ -90,20 +90,10 @@ fn key_in_dict(checker: &Checker, left: &Expr, right: &Expr, operator: CmpOp, pa } // Extract the exact range of the left and right expressions. - let left_range = parenthesized_range( - left.into(), - parent, - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(left.range()); - let right_range = parenthesized_range( - right.into(), - parent, - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(right.range()); + let left_range = + parenthesized_range(left.into(), parent, checker.tokens()).unwrap_or(left.range()); + let right_range = + parenthesized_range(right.into(), parent, checker.tokens()).unwrap_or(right.range()); let mut diagnostic = checker.report_diagnostic( InDictKeys { diff --git a/crates/ruff_linter/src/rules/flake8_type_checking/rules/type_alias_quotes.rs b/crates/ruff_linter/src/rules/flake8_type_checking/rules/type_alias_quotes.rs index 776ce1486e..c07de4b813 100644 --- a/crates/ruff_linter/src/rules/flake8_type_checking/rules/type_alias_quotes.rs +++ b/crates/ruff_linter/src/rules/flake8_type_checking/rules/type_alias_quotes.rs @@ -11,7 +11,7 @@ use crate::registry::Rule; use crate::rules::flake8_type_checking::helpers::quote_type_expression; use crate::{AlwaysFixableViolation, Edit, Fix, FixAvailability, Violation}; use ruff_python_ast::PythonVersion; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; /// ## What it does /// Checks if [PEP 613] explicit type aliases contain references to @@ -295,21 +295,20 @@ pub(crate) fn quoted_type_alias( let range = annotation_expr.range(); let mut diagnostic = checker.report_diagnostic(QuotedTypeAlias, range); let fix_string = annotation_expr.value.to_string(); + let fix_string = if (fix_string.contains('\n') || fix_string.contains('\r')) && parenthesized_range( - // Check for parenthesis outside string ("""...""") + // Check for parentheses outside the string ("""...""") annotation_expr.into(), checker.semantic().current_statement().into(), - checker.comment_ranges(), - checker.locator().contents(), + checker.source_tokens(), ) .is_none() && parenthesized_range( - // Check for parenthesis inside string """(...)""" + // Check for parentheses inside the string """(...)""" expr.into(), annotation_expr.into(), - checker.comment_ranges(), - checker.locator().contents(), + checker.tokens(), ) .is_none() { diff --git a/crates/ruff_linter/src/rules/flake8_use_pathlib/rules/path_constructor_current_directory.rs b/crates/ruff_linter/src/rules/flake8_use_pathlib/rules/path_constructor_current_directory.rs index bf5a4ed8b7..840befca76 100644 --- a/crates/ruff_linter/src/rules/flake8_use_pathlib/rules/path_constructor_current_directory.rs +++ b/crates/ruff_linter/src/rules/flake8_use_pathlib/rules/path_constructor_current_directory.rs @@ -1,10 +1,9 @@ use std::ops::Range; use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{Expr, ExprBinOp, ExprCall, Operator}; use ruff_python_semantic::SemanticModel; -use ruff_python_trivia::CommentRanges; use ruff_text_size::{Ranged, TextRange}; use crate::checkers::ast::Checker; @@ -89,11 +88,7 @@ pub(crate) fn path_constructor_current_directory( let mut diagnostic = checker.report_diagnostic(PathConstructorCurrentDirectory, arg.range()); - match parent_and_next_path_fragment_range( - checker.semantic(), - checker.comment_ranges(), - checker.source(), - ) { + match parent_and_next_path_fragment_range(checker.semantic(), checker.tokens()) { Some((parent_range, next_fragment_range)) => { let next_fragment_expr = checker.locator().slice(next_fragment_range); let call_expr = checker.locator().slice(call.range()); @@ -116,7 +111,7 @@ pub(crate) fn path_constructor_current_directory( arguments, Parentheses::Preserve, checker.source(), - checker.comment_ranges(), + checker.tokens(), )?; Ok(Fix::applicable_edit(edit, applicability(call.range()))) }), @@ -125,8 +120,7 @@ pub(crate) fn path_constructor_current_directory( fn parent_and_next_path_fragment_range( semantic: &SemanticModel, - comment_ranges: &CommentRanges, - source: &str, + tokens: &ruff_python_ast::token::Tokens, ) -> Option<(TextRange, TextRange)> { let parent = semantic.current_expression_parent()?; @@ -142,6 +136,6 @@ fn parent_and_next_path_fragment_range( Some(( parent.range(), - parenthesized_range(right.into(), parent.into(), comment_ranges, source).unwrap_or(range), + parenthesized_range(right.into(), parent.into(), tokens).unwrap_or(range), )) } diff --git a/crates/ruff_linter/src/rules/pandas_vet/rules/inplace_argument.rs b/crates/ruff_linter/src/rules/pandas_vet/rules/inplace_argument.rs index 0fa448f7c5..71e0d82db9 100644 --- a/crates/ruff_linter/src/rules/pandas_vet/rules/inplace_argument.rs +++ b/crates/ruff_linter/src/rules/pandas_vet/rules/inplace_argument.rs @@ -1,8 +1,7 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::helpers::is_const_true; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::{Tokens, parenthesized_range}; use ruff_python_ast::{self as ast, Keyword, Stmt}; -use ruff_python_trivia::CommentRanges; use ruff_text_size::Ranged; use crate::Locator; @@ -91,7 +90,7 @@ pub(crate) fn inplace_argument(checker: &Checker, call: &ast::ExprCall) { call, keyword, statement, - checker.comment_ranges(), + checker.tokens(), checker.locator(), ) { diagnostic.set_fix(fix); @@ -111,21 +110,16 @@ fn convert_inplace_argument_to_assignment( call: &ast::ExprCall, keyword: &Keyword, statement: &Stmt, - comment_ranges: &CommentRanges, + tokens: &Tokens, locator: &Locator, ) -> Option { // Add the assignment. let attr = call.func.as_attribute_expr()?; let insert_assignment = Edit::insertion( format!("{name} = ", name = locator.slice(attr.value.range())), - parenthesized_range( - call.into(), - statement.into(), - comment_ranges, - locator.contents(), - ) - .unwrap_or(call.range()) - .start(), + parenthesized_range(call.into(), statement.into(), tokens) + .unwrap_or(call.range()) + .start(), ); // Remove the `inplace` argument. @@ -134,7 +128,7 @@ fn convert_inplace_argument_to_assignment( &call.arguments, Parentheses::Preserve, locator.contents(), - comment_ranges, + tokens, ) .ok()?; diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/lambda_assignment.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/lambda_assignment.rs index a42473386b..9b437aa279 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/lambda_assignment.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/lambda_assignment.rs @@ -1,5 +1,5 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{ self as ast, Expr, ExprEllipsisLiteral, ExprLambda, Identifier, Parameter, ParameterWithDefault, Parameters, Stmt, @@ -265,29 +265,19 @@ fn replace_trailing_ellipsis_with_original_expr( stmt: &Stmt, checker: &Checker, ) -> String { - let original_expr_range = parenthesized_range( - (&lambda.body).into(), - lambda.into(), - checker.comment_ranges(), - checker.source(), - ) - .unwrap_or(lambda.body.range()); + let original_expr_range = + parenthesized_range((&lambda.body).into(), lambda.into(), checker.tokens()) + .unwrap_or(lambda.body.range()); // This prevents the autofix of introducing a syntax error if the lambda's body is an // expression spanned across multiple lines. To avoid the syntax error we preserve // the parenthesis around the body. - let original_expr_in_source = if parenthesized_range( - lambda.into(), - stmt.into(), - checker.comment_ranges(), - checker.source(), - ) - .is_some() - { - format!("({})", checker.locator().slice(original_expr_range)) - } else { - checker.locator().slice(original_expr_range).to_string() - }; + let original_expr_in_source = + if parenthesized_range(lambda.into(), stmt.into(), checker.tokens()).is_some() { + format!("({})", checker.locator().slice(original_expr_range)) + } else { + checker.locator().slice(original_expr_range).to_string() + }; let placeholder_ellipsis_start = generated.rfind("...").unwrap(); let placeholder_ellipsis_end = placeholder_ellipsis_start + "...".len(); diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/literal_comparisons.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/literal_comparisons.rs index a68e492846..5ae6fe9028 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/literal_comparisons.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/literal_comparisons.rs @@ -1,4 +1,4 @@ -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::{Tokens, parenthesized_range}; use rustc_hash::FxHashMap; use ruff_macros::{ViolationMetadata, derive_message_formats}; @@ -179,15 +179,14 @@ fn is_redundant_boolean_comparison(op: CmpOp, comparator: &Expr) -> Option fn generate_redundant_comparison( compare: &ast::ExprCompare, - comment_ranges: &ruff_python_trivia::CommentRanges, + tokens: &Tokens, source: &str, comparator: &Expr, kind: bool, needs_wrap: bool, ) -> String { - let comparator_range = - parenthesized_range(comparator.into(), compare.into(), comment_ranges, source) - .unwrap_or(comparator.range()); + let comparator_range = parenthesized_range(comparator.into(), compare.into(), tokens) + .unwrap_or(comparator.range()); let comparator_str = &source[comparator_range]; @@ -379,7 +378,7 @@ pub(crate) fn literal_comparisons(checker: &Checker, compare: &ast::ExprCompare) .copied() .collect::>(); - let comment_ranges = checker.comment_ranges(); + let tokens = checker.tokens(); let source = checker.source(); let content = match (&*compare.ops, &*compare.comparators) { @@ -387,18 +386,13 @@ pub(crate) fn literal_comparisons(checker: &Checker, compare: &ast::ExprCompare) if let Some(kind) = is_redundant_boolean_comparison(*op, &compare.left) { let needs_wrap = compare.left.range().start() != compare.range().start(); generate_redundant_comparison( - compare, - comment_ranges, - source, - comparator, - kind, - needs_wrap, + compare, tokens, source, comparator, kind, needs_wrap, ) } else if let Some(kind) = is_redundant_boolean_comparison(*op, comparator) { let needs_wrap = comparator.range().end() != compare.range().end(); generate_redundant_comparison( compare, - comment_ranges, + tokens, source, &compare.left, kind, @@ -410,7 +404,7 @@ pub(crate) fn literal_comparisons(checker: &Checker, compare: &ast::ExprCompare) &ops, &compare.comparators, compare.into(), - comment_ranges, + tokens, source, ) } @@ -420,7 +414,7 @@ pub(crate) fn literal_comparisons(checker: &Checker, compare: &ast::ExprCompare) &ops, &compare.comparators, compare.into(), - comment_ranges, + tokens, source, ), }; diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/not_tests.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/not_tests.rs index a4234093bc..0c759f9e0e 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/not_tests.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/not_tests.rs @@ -107,7 +107,7 @@ pub(crate) fn not_tests(checker: &Checker, unary_op: &ast::ExprUnaryOp) { &[CmpOp::NotIn], comparators, unary_op.into(), - checker.comment_ranges(), + checker.tokens(), checker.source(), ), unary_op.range(), @@ -127,7 +127,7 @@ pub(crate) fn not_tests(checker: &Checker, unary_op: &ast::ExprUnaryOp) { &[CmpOp::IsNot], comparators, unary_op.into(), - checker.comment_ranges(), + checker.tokens(), checker.source(), ), unary_op.range(), diff --git a/crates/ruff_linter/src/rules/pyflakes/rules/repeated_keys.rs b/crates/ruff_linter/src/rules/pyflakes/rules/repeated_keys.rs index 1acdc90138..de02e4c85a 100644 --- a/crates/ruff_linter/src/rules/pyflakes/rules/repeated_keys.rs +++ b/crates/ruff_linter/src/rules/pyflakes/rules/repeated_keys.rs @@ -3,7 +3,7 @@ use std::collections::hash_map::Entry; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::comparable::{ComparableExpr, HashableExpr}; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{self as ast, Expr}; use ruff_text_size::Ranged; @@ -193,16 +193,14 @@ pub(crate) fn repeated_keys(checker: &Checker, dict: &ast::ExprDict) { parenthesized_range( dict.value(i - 1).into(), dict.into(), - checker.comment_ranges(), - checker.locator().contents(), + checker.tokens(), ) .unwrap_or_else(|| dict.value(i - 1).range()) .end(), parenthesized_range( dict.value(i).into(), dict.into(), - checker.comment_ranges(), - checker.locator().contents(), + checker.tokens(), ) .unwrap_or_else(|| dict.value(i).range()) .end(), @@ -224,16 +222,14 @@ pub(crate) fn repeated_keys(checker: &Checker, dict: &ast::ExprDict) { parenthesized_range( dict.value(i - 1).into(), dict.into(), - checker.comment_ranges(), - checker.locator().contents(), + checker.tokens(), ) .unwrap_or_else(|| dict.value(i - 1).range()) .end(), parenthesized_range( dict.value(i).into(), dict.into(), - checker.comment_ranges(), - checker.locator().contents(), + checker.tokens(), ) .unwrap_or_else(|| dict.value(i).range()) .end(), diff --git a/crates/ruff_linter/src/rules/pyflakes/rules/unused_variable.rs b/crates/ruff_linter/src/rules/pyflakes/rules/unused_variable.rs index 810c5742b9..59dcbf5c22 100644 --- a/crates/ruff_linter/src/rules/pyflakes/rules/unused_variable.rs +++ b/crates/ruff_linter/src/rules/pyflakes/rules/unused_variable.rs @@ -2,7 +2,7 @@ use itertools::Itertools; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::helpers::contains_effect; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::token::{TokenKind, Tokens}; use ruff_python_ast::{self as ast, Stmt}; use ruff_python_semantic::Binding; @@ -172,14 +172,10 @@ fn remove_unused_variable(binding: &Binding, checker: &Checker) -> Option { { // If the expression is complex (`x = foo()`), remove the assignment, // but preserve the right-hand side. - let start = parenthesized_range( - target.into(), - statement.into(), - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(target.range()) - .start(); + let start = + parenthesized_range(target.into(), statement.into(), checker.tokens()) + .unwrap_or(target.range()) + .start(); let end = match_token_after(checker.tokens(), target.end(), |token| { token == TokenKind::Equal })? diff --git a/crates/ruff_linter/src/rules/pylint/rules/boolean_chained_comparison.rs b/crates/ruff_linter/src/rules/pylint/rules/boolean_chained_comparison.rs index 27d6d49ad5..9673524a93 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/boolean_chained_comparison.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/boolean_chained_comparison.rs @@ -2,7 +2,7 @@ use itertools::Itertools; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::{ BoolOp, CmpOp, Expr, ExprBoolOp, ExprCompare, - parenthesize::{parentheses_iterator, parenthesized_range}, + token::{parentheses_iterator, parenthesized_range}, }; use ruff_text_size::{Ranged, TextRange}; @@ -62,7 +62,7 @@ pub(crate) fn boolean_chained_comparison(checker: &Checker, expr_bool_op: &ExprB } let locator = checker.locator(); - let comment_ranges = checker.comment_ranges(); + let tokens = checker.tokens(); // retrieve all compare expressions from boolean expression let compare_expressions = expr_bool_op @@ -89,40 +89,22 @@ pub(crate) fn boolean_chained_comparison(checker: &Checker, expr_bool_op: &ExprB continue; } - let left_paren_count = parentheses_iterator( - left_compare.into(), - Some(expr_bool_op.into()), - comment_ranges, - locator.contents(), - ) - .count(); + let left_paren_count = + parentheses_iterator(left_compare.into(), Some(expr_bool_op.into()), tokens).count(); - let right_paren_count = parentheses_iterator( - right_compare.into(), - Some(expr_bool_op.into()), - comment_ranges, - locator.contents(), - ) - .count(); + let right_paren_count = + parentheses_iterator(right_compare.into(), Some(expr_bool_op.into()), tokens).count(); // Create the edit that removes the comparison operator // In `a<(b) and ((b)) "rsplit", }; - let maxsplit_argument_edit = fix::edits::add_argument( - "maxsplit=1", - arguments, - checker.comment_ranges(), - checker.locator().contents(), - ); + let maxsplit_argument_edit = + fix::edits::add_argument("maxsplit=1", arguments, checker.tokens()); // Only change `actual_split_type` if it doesn't match `suggested_split_type` let split_type_edit: Option = if actual_split_type == suggested_split_type { diff --git a/crates/ruff_linter/src/rules/pylint/rules/non_augmented_assignment.rs b/crates/ruff_linter/src/rules/pylint/rules/non_augmented_assignment.rs index 7423c2dc76..cbe51cdd8d 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/non_augmented_assignment.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/non_augmented_assignment.rs @@ -2,7 +2,7 @@ use ast::Expr; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast as ast; use ruff_python_ast::comparable::ComparableExpr; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{ExprBinOp, ExprRef, Operator}; use ruff_text_size::{Ranged, TextRange}; @@ -150,12 +150,10 @@ fn augmented_assignment( let right_operand_ref = ExprRef::from(right_operand); let parent = original_expr.into(); - let comment_ranges = checker.comment_ranges(); - let source = checker.source(); + let tokens = checker.tokens(); let right_operand_range = - parenthesized_range(right_operand_ref, parent, comment_ranges, source) - .unwrap_or(right_operand.range()); + parenthesized_range(right_operand_ref, parent, tokens).unwrap_or(right_operand.range()); let right_operand_expr = locator.slice(right_operand_range); let target_expr = locator.slice(target); diff --git a/crates/ruff_linter/src/rules/pylint/rules/subprocess_run_without_check.rs b/crates/ruff_linter/src/rules/pylint/rules/subprocess_run_without_check.rs index 0ed569eef8..21a4643f39 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/subprocess_run_without_check.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/subprocess_run_without_check.rs @@ -75,12 +75,7 @@ pub(crate) fn subprocess_run_without_check(checker: &Checker, call: &ast::ExprCa let mut diagnostic = checker.report_diagnostic(SubprocessRunWithoutCheck, call.func.range()); diagnostic.set_fix(Fix::applicable_edit( - add_argument( - "check=False", - &call.arguments, - checker.comment_ranges(), - checker.locator().contents(), - ), + add_argument("check=False", &call.arguments, checker.tokens()), // If the function call contains `**kwargs`, mark the fix as unsafe. if call .arguments diff --git a/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs b/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs index 7d9cd12506..316d2e7f54 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs @@ -1,8 +1,7 @@ use std::fmt::{Display, Formatter}; use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::name::QualifiedName; -use ruff_python_ast::{self as ast, Expr}; +use ruff_python_ast::{self as ast, Expr, name::QualifiedName}; use ruff_python_semantic::SemanticModel; use ruff_python_semantic::analyze::typing; use ruff_text_size::{Ranged, TextRange}; @@ -193,8 +192,7 @@ fn generate_keyword_fix(checker: &Checker, call: &ast::ExprCall) -> Fix { })) ), &call.arguments, - checker.comment_ranges(), - checker.locator().contents(), + checker.tokens(), )) } diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/pep695/non_pep695_generic_class.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/pep695/non_pep695_generic_class.rs index b0d273b7c4..02f8cdd1e8 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/pep695/non_pep695_generic_class.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/pep695/non_pep695_generic_class.rs @@ -204,7 +204,7 @@ pub(crate) fn non_pep695_generic_class(checker: &Checker, class_def: &StmtClassD arguments, Parentheses::Remove, checker.source(), - checker.comment_ranges(), + checker.tokens(), )?; Ok(Fix::unsafe_edits( Edit::insertion(type_params.to_string(), name.end()), diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/pep695/non_pep695_type_alias.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/pep695/non_pep695_type_alias.rs index 43e3ec8536..6b10c3bc07 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/pep695/non_pep695_type_alias.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/pep695/non_pep695_type_alias.rs @@ -2,7 +2,7 @@ use itertools::Itertools; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::name::Name; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::visitor::Visitor; use ruff_python_ast::{Expr, ExprCall, ExprName, Keyword, StmtAnnAssign, StmtAssign, StmtRef}; use ruff_text_size::{Ranged, TextRange}; @@ -261,11 +261,11 @@ fn create_diagnostic( type_alias_kind: TypeAliasKind, ) { let source = checker.source(); + let tokens = checker.tokens(); let comment_ranges = checker.comment_ranges(); let range_with_parentheses = - parenthesized_range(value.into(), stmt.into(), comment_ranges, source) - .unwrap_or(value.range()); + parenthesized_range(value.into(), stmt.into(), tokens).unwrap_or(value.range()); let content = format!( "type {name}{type_params} = {value}", diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/replace_stdout_stderr.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/replace_stdout_stderr.rs index 7c5dd2f027..6ce403647d 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/replace_stdout_stderr.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/replace_stdout_stderr.rs @@ -1,9 +1,8 @@ use anyhow::Result; use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::{self as ast, Keyword}; +use ruff_python_ast::{self as ast, Keyword, token::Tokens}; use ruff_python_semantic::Modules; -use ruff_python_trivia::CommentRanges; use ruff_text_size::Ranged; use crate::checkers::ast::Checker; @@ -104,7 +103,7 @@ pub(crate) fn replace_stdout_stderr(checker: &Checker, call: &ast::ExprCall) { stderr, call, checker.locator().contents(), - checker.comment_ranges(), + checker.tokens(), ) }); } @@ -117,7 +116,7 @@ fn generate_fix( stderr: &Keyword, call: &ast::ExprCall, source: &str, - comment_ranges: &CommentRanges, + tokens: &Tokens, ) -> Result { let (first, second) = if stdout.start() < stderr.start() { (stdout, stderr) @@ -132,7 +131,7 @@ fn generate_fix( &call.arguments, Parentheses::Preserve, source, - comment_ranges, + tokens, )?], )) } diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/replace_universal_newlines.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/replace_universal_newlines.rs index 97000fa75f..b7b13c7b66 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/replace_universal_newlines.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/replace_universal_newlines.rs @@ -78,7 +78,7 @@ pub(crate) fn replace_universal_newlines(checker: &Checker, call: &ast::ExprCall &call.arguments, Parentheses::Preserve, checker.locator().contents(), - checker.comment_ranges(), + checker.tokens(), ) .map(Fix::safe_edit) }); diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs index 01cfe4fb03..792365042f 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs @@ -188,7 +188,7 @@ pub(crate) fn unnecessary_encode_utf8(checker: &Checker, call: &ast::ExprCall) { &call.arguments, Parentheses::Preserve, checker.locator().contents(), - checker.comment_ranges(), + checker.tokens(), ) .map(Fix::safe_edit) }); @@ -206,7 +206,7 @@ pub(crate) fn unnecessary_encode_utf8(checker: &Checker, call: &ast::ExprCall) { &call.arguments, Parentheses::Preserve, checker.locator().contents(), - checker.comment_ranges(), + checker.tokens(), ) .map(Fix::safe_edit) }); @@ -231,7 +231,7 @@ pub(crate) fn unnecessary_encode_utf8(checker: &Checker, call: &ast::ExprCall) { &call.arguments, Parentheses::Preserve, checker.locator().contents(), - checker.comment_ranges(), + checker.tokens(), ) .map(Fix::safe_edit) }); @@ -249,7 +249,7 @@ pub(crate) fn unnecessary_encode_utf8(checker: &Checker, call: &ast::ExprCall) { &call.arguments, Parentheses::Preserve, checker.locator().contents(), - checker.comment_ranges(), + checker.tokens(), ) .map(Fix::safe_edit) }); diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/useless_class_metaclass_type.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/useless_class_metaclass_type.rs index 75bbe20eca..20d3f64461 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/useless_class_metaclass_type.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/useless_class_metaclass_type.rs @@ -70,7 +70,7 @@ pub(crate) fn useless_class_metaclass_type(checker: &Checker, class_def: &StmtCl arguments, Parentheses::Remove, checker.locator().contents(), - checker.comment_ranges(), + checker.tokens(), )?; let range = edit.range(); diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/useless_object_inheritance.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/useless_object_inheritance.rs index 4a5789c78f..a1b0d900f8 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/useless_object_inheritance.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/useless_object_inheritance.rs @@ -73,7 +73,7 @@ pub(crate) fn useless_object_inheritance(checker: &Checker, class_def: &ast::Stm arguments, Parentheses::Remove, checker.locator().contents(), - checker.comment_ranges(), + checker.tokens(), )?; let range = edit.range(); diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/yield_in_for_loop.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/yield_in_for_loop.rs index 7fec2b7d79..55f2afc89a 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/yield_in_for_loop.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/yield_in_for_loop.rs @@ -1,5 +1,5 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{self as ast, Expr, Stmt}; use ruff_text_size::Ranged; @@ -139,13 +139,8 @@ pub(crate) fn yield_in_for_loop(checker: &Checker, stmt_for: &ast::StmtFor) { let mut diagnostic = checker.report_diagnostic(YieldInForLoop, stmt_for.range()); let contents = checker.locator().slice( - parenthesized_range( - iter.as_ref().into(), - stmt_for.into(), - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(iter.range()), + parenthesized_range(iter.as_ref().into(), stmt_for.into(), checker.tokens()) + .unwrap_or(iter.range()), ); let contents = if iter.as_tuple_expr().is_some_and(|it| !it.parenthesized) { format!("yield from ({contents})") diff --git a/crates/ruff_linter/src/rules/refurb/helpers.rs b/crates/ruff_linter/src/rules/refurb/helpers.rs index 0a09d70aba..a6871f0497 100644 --- a/crates/ruff_linter/src/rules/refurb/helpers.rs +++ b/crates/ruff_linter/src/rules/refurb/helpers.rs @@ -1,7 +1,7 @@ use std::borrow::Cow; use ruff_python_ast::PythonVersion; -use ruff_python_ast::{self as ast, Expr, name::Name, parenthesize::parenthesized_range}; +use ruff_python_ast::{self as ast, Expr, name::Name, token::parenthesized_range}; use ruff_python_codegen::Generator; use ruff_python_semantic::{BindingId, ResolvedReference, SemanticModel}; use ruff_text_size::{Ranged, TextRange}; @@ -330,12 +330,8 @@ pub(super) fn parenthesize_loop_iter_if_necessary<'a>( let locator = checker.locator(); let iter = for_stmt.iter.as_ref(); - let original_parenthesized_range = parenthesized_range( - iter.into(), - for_stmt.into(), - checker.comment_ranges(), - checker.source(), - ); + let original_parenthesized_range = + parenthesized_range(iter.into(), for_stmt.into(), checker.tokens()); if let Some(range) = original_parenthesized_range { return Cow::Borrowed(locator.slice(range)); diff --git a/crates/ruff_linter/src/rules/refurb/rules/fromisoformat_replace_z.rs b/crates/ruff_linter/src/rules/refurb/rules/fromisoformat_replace_z.rs index 3622148fbf..b2b3193d9e 100644 --- a/crates/ruff_linter/src/rules/refurb/rules/fromisoformat_replace_z.rs +++ b/crates/ruff_linter/src/rules/refurb/rules/fromisoformat_replace_z.rs @@ -1,5 +1,5 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{ Expr, ExprAttribute, ExprBinOp, ExprCall, ExprStringLiteral, ExprSubscript, ExprUnaryOp, Number, Operator, PythonVersion, UnaryOp, @@ -112,8 +112,7 @@ pub(crate) fn fromisoformat_replace_z(checker: &Checker, call: &ExprCall) { let value_full_range = parenthesized_range( replace_time_zone.date.into(), replace_time_zone.parent.into(), - checker.comment_ranges(), - checker.source(), + checker.tokens(), ) .unwrap_or(replace_time_zone.date.range()); diff --git a/crates/ruff_linter/src/rules/refurb/rules/if_exp_instead_of_or_operator.rs b/crates/ruff_linter/src/rules/refurb/rules/if_exp_instead_of_or_operator.rs index fa660587ef..f4f4d1f7b7 100644 --- a/crates/ruff_linter/src/rules/refurb/rules/if_exp_instead_of_or_operator.rs +++ b/crates/ruff_linter/src/rules/refurb/rules/if_exp_instead_of_or_operator.rs @@ -5,8 +5,7 @@ use ruff_python_ast as ast; use ruff_python_ast::Expr; use ruff_python_ast::comparable::ComparableExpr; use ruff_python_ast::helpers::contains_effect; -use ruff_python_ast::parenthesize::parenthesized_range; -use ruff_python_trivia::CommentRanges; +use ruff_python_ast::token::{Tokens, parenthesized_range}; use ruff_text_size::Ranged; use crate::Locator; @@ -76,8 +75,8 @@ pub(crate) fn if_exp_instead_of_or_operator(checker: &Checker, if_expr: &ast::Ex Edit::range_replacement( format!( "{} or {}", - parenthesize_test(test, if_expr, checker.comment_ranges(), checker.locator()), - parenthesize_test(orelse, if_expr, checker.comment_ranges(), checker.locator()), + parenthesize_test(test, if_expr, checker.tokens(), checker.locator()), + parenthesize_test(orelse, if_expr, checker.tokens(), checker.locator()), ), if_expr.range(), ), @@ -99,15 +98,10 @@ pub(crate) fn if_exp_instead_of_or_operator(checker: &Checker, if_expr: &ast::Ex fn parenthesize_test<'a>( expr: &Expr, if_expr: &ast::ExprIf, - comment_ranges: &CommentRanges, + tokens: &Tokens, locator: &Locator<'a>, ) -> Cow<'a, str> { - if let Some(range) = parenthesized_range( - expr.into(), - if_expr.into(), - comment_ranges, - locator.contents(), - ) { + if let Some(range) = parenthesized_range(expr.into(), if_expr.into(), tokens) { Cow::Borrowed(locator.slice(range)) } else if matches!(expr, Expr::If(_) | Expr::Lambda(_) | Expr::Named(_)) { Cow::Owned(format!("({})", locator.slice(expr.range()))) diff --git a/crates/ruff_linter/src/rules/refurb/rules/readlines_in_for.rs b/crates/ruff_linter/src/rules/refurb/rules/readlines_in_for.rs index a6ea1eb570..943a013cbb 100644 --- a/crates/ruff_linter/src/rules/refurb/rules/readlines_in_for.rs +++ b/crates/ruff_linter/src/rules/refurb/rules/readlines_in_for.rs @@ -1,6 +1,6 @@ use ruff_diagnostics::Applicability; use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{Comprehension, Expr, StmtFor}; use ruff_python_semantic::analyze::typing; use ruff_python_semantic::analyze::typing::is_io_base_expr; @@ -104,8 +104,7 @@ fn readlines_in_iter(checker: &Checker, iter_expr: &Expr) { let deletion_range = if let Some(parenthesized_range) = parenthesized_range( expr_attr.value.as_ref().into(), expr_attr.into(), - checker.comment_ranges(), - checker.source(), + checker.tokens(), ) { expr_call.range().add_start(parenthesized_range.len()) } else { diff --git a/crates/ruff_linter/src/rules/refurb/rules/redundant_log_base.rs b/crates/ruff_linter/src/rules/refurb/rules/redundant_log_base.rs index a54bd261d1..35774cde28 100644 --- a/crates/ruff_linter/src/rules/refurb/rules/redundant_log_base.rs +++ b/crates/ruff_linter/src/rules/refurb/rules/redundant_log_base.rs @@ -1,7 +1,7 @@ use anyhow::Result; use ruff_diagnostics::Applicability; use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{self as ast, Expr, Number}; use ruff_text_size::Ranged; @@ -152,13 +152,8 @@ fn generate_fix(checker: &Checker, call: &ast::ExprCall, base: Base, arg: &Expr) checker.semantic(), )?; - let arg_range = parenthesized_range( - arg.into(), - call.into(), - checker.comment_ranges(), - checker.source(), - ) - .unwrap_or(arg.range()); + let arg_range = + parenthesized_range(arg.into(), call.into(), checker.tokens()).unwrap_or(arg.range()); let arg_str = checker.locator().slice(arg_range); Ok(Fix::applicable_edits( diff --git a/crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs b/crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs index 1df2fdde78..04b00f07f7 100644 --- a/crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs +++ b/crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs @@ -95,7 +95,7 @@ pub(crate) fn single_item_membership_test( &[membership_test.replacement_op()], std::slice::from_ref(item), expr.into(), - checker.comment_ranges(), + checker.tokens(), checker.source(), ), expr.range(), diff --git a/crates/ruff_linter/src/rules/ruff/rules/class_with_mixed_type_vars.rs b/crates/ruff_linter/src/rules/ruff/rules/class_with_mixed_type_vars.rs index 4178217718..7b023de830 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/class_with_mixed_type_vars.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/class_with_mixed_type_vars.rs @@ -163,7 +163,7 @@ fn convert_type_vars( class_arguments, Parentheses::Remove, source, - checker.comment_ranges(), + checker.tokens(), )?; let replace_type_params = Edit::range_replacement(new_type_params.to_string(), type_params.range); diff --git a/crates/ruff_linter/src/rules/ruff/rules/default_factory_kwarg.rs b/crates/ruff_linter/src/rules/ruff/rules/default_factory_kwarg.rs index ea792ce97c..70ef8cb6d4 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/default_factory_kwarg.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/default_factory_kwarg.rs @@ -3,8 +3,8 @@ use anyhow::Result; use ast::Keyword; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::helpers::is_constant; +use ruff_python_ast::token::Tokens; use ruff_python_ast::{self as ast, Expr}; -use ruff_python_trivia::CommentRanges; use ruff_text_size::Ranged; use crate::Locator; @@ -108,9 +108,8 @@ pub(crate) fn default_factory_kwarg(checker: &Checker, call: &ast::ExprCall) { }, call.range(), ); - diagnostic.try_set_fix(|| { - convert_to_positional(call, keyword, checker.locator(), checker.comment_ranges()) - }); + diagnostic + .try_set_fix(|| convert_to_positional(call, keyword, checker.locator(), checker.tokens())); } /// Returns `true` if a value is definitively not callable (e.g., `1` or `[]`). @@ -136,7 +135,7 @@ fn convert_to_positional( call: &ast::ExprCall, default_factory: &Keyword, locator: &Locator, - comment_ranges: &CommentRanges, + tokens: &Tokens, ) -> Result { if call.arguments.len() == 1 { // Ex) `defaultdict(default_factory=list)` @@ -153,7 +152,7 @@ fn convert_to_positional( &call.arguments, Parentheses::Preserve, locator.contents(), - comment_ranges, + tokens, )?; // Second, insert the value as the first positional argument. diff --git a/crates/ruff_linter/src/rules/ruff/rules/falsy_dict_get_fallback.rs b/crates/ruff_linter/src/rules/ruff/rules/falsy_dict_get_fallback.rs index a19a9c451a..de3c072bdd 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/falsy_dict_get_fallback.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/falsy_dict_get_fallback.rs @@ -128,7 +128,7 @@ pub(crate) fn falsy_dict_get_fallback(checker: &Checker, expr: &Expr) { &call.arguments, Parentheses::Preserve, checker.locator().contents(), - checker.comment_ranges(), + checker.tokens(), ) .map(|edit| Fix::applicable_edit(edit, applicability)) }); diff --git a/crates/ruff_linter/src/rules/ruff/rules/parenthesize_chained_operators.rs b/crates/ruff_linter/src/rules/ruff/rules/parenthesize_chained_operators.rs index 7433f63f2b..6e2351aafb 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/parenthesize_chained_operators.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/parenthesize_chained_operators.rs @@ -1,6 +1,6 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast as ast; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_text_size::Ranged; use crate::checkers::ast::Checker; @@ -77,14 +77,7 @@ pub(crate) fn parenthesize_chained_logical_operators(checker: &Checker, expr: &a ) => { let locator = checker.locator(); let source_range = bool_op.range(); - if parenthesized_range( - bool_op.into(), - expr.into(), - checker.comment_ranges(), - locator.contents(), - ) - .is_none() - { + if parenthesized_range(bool_op.into(), expr.into(), checker.tokens()).is_none() { let new_source = format!("({})", locator.slice(source_range)); let edit = Edit::range_replacement(new_source, source_range); checker diff --git a/crates/ruff_linter/src/rules/ruff/rules/post_init_default.rs b/crates/ruff_linter/src/rules/ruff/rules/post_init_default.rs index 36941a98c9..2ec2472262 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/post_init_default.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/post_init_default.rs @@ -2,7 +2,7 @@ use anyhow::Context; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast as ast; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_semantic::{Scope, ScopeKind}; use ruff_python_trivia::{indentation_at_offset, textwrap}; use ruff_source_file::LineRanges; @@ -159,8 +159,7 @@ fn use_initvar( let default_loc = parenthesized_range( default.into(), parameter_with_default.into(), - checker.comment_ranges(), - checker.source(), + checker.tokens(), ) .unwrap_or(default.range()); diff --git a/crates/ruff_linter/src/rules/ruff/rules/quadratic_list_summation.rs b/crates/ruff_linter/src/rules/ruff/rules/quadratic_list_summation.rs index d4e062ad71..a38fd2fd6d 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/quadratic_list_summation.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/quadratic_list_summation.rs @@ -2,7 +2,7 @@ use anyhow::Result; use itertools::Itertools; use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{self as ast, Arguments, Expr}; use ruff_python_semantic::SemanticModel; use ruff_text_size::Ranged; @@ -116,13 +116,8 @@ fn convert_to_reduce(iterable: &Expr, call: &ast::ExprCall, checker: &Checker) - )?; let iterable = checker.locator().slice( - parenthesized_range( - iterable.into(), - (&call.arguments).into(), - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(iterable.range()), + parenthesized_range(iterable.into(), (&call.arguments).into(), checker.tokens()) + .unwrap_or(iterable.range()), ); Ok(Fix::unsafe_edits( diff --git a/crates/ruff_linter/src/rules/ruff/rules/starmap_zip.rs b/crates/ruff_linter/src/rules/ruff/rules/starmap_zip.rs index e9ed8d31bb..79881b8abc 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/starmap_zip.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/starmap_zip.rs @@ -1,7 +1,7 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::PythonVersion; use ruff_python_ast::token::TokenKind; -use ruff_python_ast::{Expr, ExprCall, parenthesize::parenthesized_range}; +use ruff_python_ast::{Expr, ExprCall, token::parenthesized_range}; use ruff_text_size::{Ranged, TextRange}; use crate::checkers::ast::Checker; @@ -124,13 +124,8 @@ fn replace_with_map(starmap: &ExprCall, zip: &ExprCall, checker: &Checker) -> Op let mut remove_zip = vec![]; - let full_zip_range = parenthesized_range( - zip.into(), - starmap.into(), - checker.comment_ranges(), - checker.source(), - ) - .unwrap_or(zip.range()); + let full_zip_range = + parenthesized_range(zip.into(), starmap.into(), checker.tokens()).unwrap_or(zip.range()); // Delete any parentheses around the `zip` call to prevent that the argument turns into a tuple. remove_zip.push(Edit::range_deletion(TextRange::new( @@ -138,13 +133,8 @@ fn replace_with_map(starmap: &ExprCall, zip: &ExprCall, checker: &Checker) -> Op zip.start(), ))); - let full_zip_func_range = parenthesized_range( - (&zip.func).into(), - zip.into(), - checker.comment_ranges(), - checker.source(), - ) - .unwrap_or(zip.func.range()); + let full_zip_func_range = parenthesized_range((&zip.func).into(), zip.into(), checker.tokens()) + .unwrap_or(zip.func.range()); // Delete the `zip` callee remove_zip.push(Edit::range_deletion(full_zip_func_range)); diff --git a/crates/ruff_linter/src/rules/ruff/rules/unnecessary_cast_to_int.rs b/crates/ruff_linter/src/rules/ruff/rules/unnecessary_cast_to_int.rs index b3c34c29e0..453aa07801 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/unnecessary_cast_to_int.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/unnecessary_cast_to_int.rs @@ -1,5 +1,5 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::{Tokens, parenthesized_range}; use ruff_python_ast::{Arguments, Expr, ExprCall}; use ruff_python_semantic::SemanticModel; use ruff_python_semantic::analyze::type_inference::{NumberLike, PythonType, ResolvedPythonType}; @@ -86,6 +86,7 @@ pub(crate) fn unnecessary_cast_to_int(checker: &Checker, call: &ExprCall) { applicability, checker.semantic(), checker.locator(), + checker.tokens(), checker.comment_ranges(), checker.source(), ); @@ -95,27 +96,26 @@ pub(crate) fn unnecessary_cast_to_int(checker: &Checker, call: &ExprCall) { } /// Creates a fix that replaces `int(expression)` with `expression`. +#[allow(clippy::too_many_arguments)] fn unwrap_int_expression( call: &ExprCall, argument: &Expr, applicability: Applicability, semantic: &SemanticModel, locator: &Locator, + tokens: &Tokens, comment_ranges: &CommentRanges, source: &str, ) -> Fix { - let content = if let Some(range) = parenthesized_range( - argument.into(), - (&call.arguments).into(), - comment_ranges, - source, - ) { + let content = if let Some(range) = + parenthesized_range(argument.into(), (&call.arguments).into(), tokens) + { locator.slice(range).to_string() } else { let parenthesize = semantic.current_expression_parent().is_some() || argument.is_named_expr() || locator.count_lines(argument.range()) > 0; - if parenthesize && !has_own_parentheses(argument, comment_ranges, source) { + if parenthesize && !has_own_parentheses(argument, tokens, source) { format!("({})", locator.slice(argument.range())) } else { locator.slice(argument.range()).to_string() @@ -255,7 +255,7 @@ fn round_applicability(arguments: &Arguments, semantic: &SemanticModel) -> Optio } /// Returns `true` if the given [`Expr`] has its own parentheses (e.g., `()`, `[]`, `{}`). -fn has_own_parentheses(expr: &Expr, comment_ranges: &CommentRanges, source: &str) -> bool { +fn has_own_parentheses(expr: &Expr, tokens: &Tokens, source: &str) -> bool { match expr { Expr::ListComp(_) | Expr::SetComp(_) @@ -276,14 +276,10 @@ fn has_own_parentheses(expr: &Expr, comment_ranges: &CommentRanges, source: &str // f // (10) // ``` - let func_end = parenthesized_range( - call_expr.func.as_ref().into(), - call_expr.into(), - comment_ranges, - source, - ) - .unwrap_or(call_expr.func.range()) - .end(); + let func_end = + parenthesized_range(call_expr.func.as_ref().into(), call_expr.into(), tokens) + .unwrap_or(call_expr.func.range()) + .end(); lines_after_ignoring_trivia(func_end, source) == 0 } Expr::Subscript(subscript_expr) => { @@ -291,8 +287,7 @@ fn has_own_parentheses(expr: &Expr, comment_ranges: &CommentRanges, source: &str let subscript_end = parenthesized_range( subscript_expr.value.as_ref().into(), subscript_expr.into(), - comment_ranges, - source, + tokens, ) .unwrap_or(subscript_expr.value.range()) .end(); diff --git a/crates/ruff_linter/src/rules/ruff/rules/unnecessary_key_check.rs b/crates/ruff_linter/src/rules/ruff/rules/unnecessary_key_check.rs index 7c13fb3d1c..502391dcf2 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/unnecessary_key_check.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/unnecessary_key_check.rs @@ -3,7 +3,7 @@ use ruff_python_ast::{self as ast, BoolOp, CmpOp, Expr}; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::helpers::contains_effect; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_text_size::Ranged; use crate::checkers::ast::Checker; @@ -108,22 +108,12 @@ pub(crate) fn unnecessary_key_check(checker: &Checker, expr: &Expr) { format!( "{}.get({})", checker.locator().slice( - parenthesized_range( - obj_right.into(), - right.into(), - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(obj_right.range()) + parenthesized_range(obj_right.into(), right.into(), checker.tokens(),) + .unwrap_or(obj_right.range()) ), checker.locator().slice( - parenthesized_range( - key_right.into(), - right.into(), - checker.comment_ranges(), - checker.locator().contents(), - ) - .unwrap_or(key_right.range()) + parenthesized_range(key_right.into(), right.into(), checker.tokens(),) + .unwrap_or(key_right.range()) ), ), expr.range(), diff --git a/crates/ruff_linter/src/rules/ruff/rules/unnecessary_literal_within_deque_call.rs b/crates/ruff_linter/src/rules/ruff/rules/unnecessary_literal_within_deque_call.rs index fd31765715..b322d57fd6 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/unnecessary_literal_within_deque_call.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/unnecessary_literal_within_deque_call.rs @@ -2,7 +2,7 @@ use ruff_diagnostics::{Applicability, Edit}; use ruff_macros::{ViolationMetadata, derive_message_formats}; use ruff_python_ast::helpers::is_empty_f_string; -use ruff_python_ast::parenthesize::parenthesized_range; +use ruff_python_ast::token::parenthesized_range; use ruff_python_ast::{self as ast, Expr}; use ruff_text_size::Ranged; @@ -140,31 +140,19 @@ fn fix_unnecessary_literal_in_deque( // call. otherwise, we only delete the `iterable` argument and leave the others untouched. let edit = if let Some(maxlen) = maxlen { let deque_name = checker.locator().slice( - parenthesized_range( - deque.func.as_ref().into(), - deque.into(), - checker.comment_ranges(), - checker.source(), - ) - .unwrap_or(deque.func.range()), + parenthesized_range(deque.func.as_ref().into(), deque.into(), checker.tokens()) + .unwrap_or(deque.func.range()), ); let len_str = checker.locator().slice(maxlen); let deque_str = format!("{deque_name}(maxlen={len_str})"); Edit::range_replacement(deque_str, deque.range) } else { - let range = parenthesized_range( - iterable.value().into(), - (&deque.arguments).into(), - checker.comment_ranges(), - checker.source(), - ) - .unwrap_or(iterable.range()); remove_argument( - &range, + &iterable, &deque.arguments, Parentheses::Preserve, checker.source(), - checker.comment_ranges(), + checker.tokens(), )? }; let has_comments = checker.comment_ranges().intersects(edit.range()); diff --git a/crates/ruff_python_ast/src/helpers.rs b/crates/ruff_python_ast/src/helpers.rs index 4879e04780..9680d03ec3 100644 --- a/crates/ruff_python_ast/src/helpers.rs +++ b/crates/ruff_python_ast/src/helpers.rs @@ -3,13 +3,14 @@ use std::path::Path; use rustc_hash::FxHashMap; -use ruff_python_trivia::{CommentRanges, SimpleTokenKind, SimpleTokenizer, indentation_at_offset}; +use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer, indentation_at_offset}; use ruff_source_file::LineRanges; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use crate::name::{Name, QualifiedName, QualifiedNameBuilder}; -use crate::parenthesize::parenthesized_range; use crate::statement_visitor::StatementVisitor; +use crate::token::Tokens; +use crate::token::parenthesized_range; use crate::visitor::Visitor; use crate::{ self as ast, Arguments, AtomicNodeIndex, CmpOp, DictItem, ExceptHandler, Expr, ExprNoneLiteral, @@ -1474,7 +1475,7 @@ pub fn generate_comparison( ops: &[CmpOp], comparators: &[Expr], parent: AnyNodeRef, - comment_ranges: &CommentRanges, + tokens: &Tokens, source: &str, ) -> String { let start = left.start(); @@ -1483,8 +1484,7 @@ pub fn generate_comparison( // Add the left side of the comparison. contents.push_str( - &source[parenthesized_range(left.into(), parent, comment_ranges, source) - .unwrap_or(left.range())], + &source[parenthesized_range(left.into(), parent, tokens).unwrap_or(left.range())], ); for (op, comparator) in ops.iter().zip(comparators) { @@ -1504,7 +1504,7 @@ pub fn generate_comparison( // Add the right side of the comparison. contents.push_str( - &source[parenthesized_range(comparator.into(), parent, comment_ranges, source) + &source[parenthesized_range(comparator.into(), parent, tokens) .unwrap_or(comparator.range())], ); }