From 03be8b9454fb59be04594684bc356ede02cd326c Mon Sep 17 00:00:00 2001 From: Denys Zhak Date: Tue, 9 Dec 2025 02:54:33 +0000 Subject: [PATCH] feat: rollback to SimpleTokenizer in remove_argument --- crates/ruff_linter/src/fix/edits.rs | 24 ++++++++--------- .../rules/fastapi_redundant_response_model.rs | 1 + .../rules/exc_info_outside_except_handler.rs | 8 +++++- .../rules/unnecessary_dict_kwargs.rs | 1 + .../rules/unnecessary_range_start.rs | 1 + .../rules/generic_not_last_base_class.rs | 9 ++++++- .../flake8_pytest_style/rules/fixture.rs | 1 + .../path_constructor_current_directory.rs | 8 +++++- .../pandas_vet/rules/inplace_argument.rs | 10 +++++-- .../src/rules/pylint/rules/duplicate_bases.rs | 27 +++++++++++-------- .../rules/pep695/non_pep695_generic_class.rs | 1 + .../pyupgrade/rules/replace_stdout_stderr.rs | 12 ++++++++- .../rules/replace_universal_newlines.rs | 1 + .../rules/unnecessary_encode_utf8.rs | 4 +++ .../rules/useless_class_metaclass_type.rs | 9 +++++-- .../rules/useless_object_inheritance.rs | 8 +++++- .../ruff/rules/class_with_mixed_type_vars.rs | 1 + .../rules/ruff/rules/default_factory_kwarg.rs | 1 + .../ruff/rules/falsy_dict_get_fallback.rs | 1 + .../unnecessary_literal_within_deque_call.rs | 1 + 20 files changed, 97 insertions(+), 32 deletions(-) diff --git a/crates/ruff_linter/src/fix/edits.rs b/crates/ruff_linter/src/fix/edits.rs index 07c96330f9..6dcd83454b 100644 --- a/crates/ruff_linter/src/fix/edits.rs +++ b/crates/ruff_linter/src/fix/edits.rs @@ -8,7 +8,9 @@ use ruff_python_ast::{self as ast, Arguments, ExceptHandler, Expr, ExprList, Par use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; use ruff_python_trivia::textwrap::dedent_to; -use ruff_python_trivia::{PythonWhitespace, has_leading_content, is_python_whitespace}; +use ruff_python_trivia::{ + PythonWhitespace, SimpleTokenKind, SimpleTokenizer, has_leading_content, is_python_whitespace, +}; use ruff_source_file::{LineRanges, NewlineWithTrailingNewline, UniversalNewlines}; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; @@ -205,6 +207,7 @@ pub(crate) fn remove_argument( argument: &T, arguments: &Arguments, parentheses: Parentheses, + source: &str, tokens: &Tokens, ) -> Result { // Partition into arguments before and after the argument to remove. @@ -226,20 +229,17 @@ pub(crate) fn remove_argument( if !after.is_empty() { // Case 1: argument or keyword is _not_ the last node, so delete from the start of the // argument to the end of the subsequent comma. - let mut tokens_after = tokens.after(argument.end()).iter(); + let mut tokenizer = SimpleTokenizer::starts_at(argument.end(), source); // Find the trailing comma. - tokens_after - .find(|token| token.kind() == TokenKind::Comma) + tokenizer + .find(|token| token.kind == SimpleTokenKind::Comma) .context("Unable to find trailing comma")?; // Find the next non-whitespace token. - let next = tokens_after + let next = tokenizer .find(|token| { - !matches!( - token.kind(), - TokenKind::Newline | TokenKind::NonLogicalNewline - ) + token.kind != SimpleTokenKind::Whitespace && token.kind != SimpleTokenKind::Newline }) .context("Unable to find next token")?; @@ -247,11 +247,11 @@ pub(crate) fn remove_argument( } else if let Some(previous) = before.iter().map(Ranged::end).max() { // Case 2: argument or keyword is the last node, so delete from the start of the // previous comma to the end of the argument. - let mut tokens_after = tokens.after(previous).iter(); + let mut tokenizer = SimpleTokenizer::starts_at(previous, source); // Find the trailing comma. - let comma = tokens_after - .find(|token| token.kind() == TokenKind::Comma) + let comma = tokenizer + .find(|token| token.kind == SimpleTokenKind::Comma) .context("Unable to find trailing comma")?; Ok(Edit::deletion(comma.start(), parenthesized_range.end())) diff --git a/crates/ruff_linter/src/rules/fastapi/rules/fastapi_redundant_response_model.rs b/crates/ruff_linter/src/rules/fastapi/rules/fastapi_redundant_response_model.rs index 0686ed68d3..d31ffbf61e 100644 --- a/crates/ruff_linter/src/rules/fastapi/rules/fastapi_redundant_response_model.rs +++ b/crates/ruff_linter/src/rules/fastapi/rules/fastapi_redundant_response_model.rs @@ -91,6 +91,7 @@ pub(crate) fn fastapi_redundant_response_model(checker: &Checker, function_def: response_model_arg, &call.arguments, Parentheses::Preserve, + checker.source(), checker.tokens(), ) .map(Fix::unsafe_edit) diff --git a/crates/ruff_linter/src/rules/flake8_logging/rules/exc_info_outside_except_handler.rs b/crates/ruff_linter/src/rules/flake8_logging/rules/exc_info_outside_except_handler.rs index 819991ac51..7d18897708 100644 --- a/crates/ruff_linter/src/rules/flake8_logging/rules/exc_info_outside_except_handler.rs +++ b/crates/ruff_linter/src/rules/flake8_logging/rules/exc_info_outside_except_handler.rs @@ -115,7 +115,13 @@ pub(crate) fn exc_info_outside_except_handler(checker: &Checker, call: &ExprCall let mut diagnostic = checker.report_diagnostic(ExcInfoOutsideExceptHandler, exc_info.range); diagnostic.try_set_fix(|| { - let edit = remove_argument(exc_info, arguments, Parentheses::Preserve, checker.tokens())?; + let edit = remove_argument( + exc_info, + arguments, + Parentheses::Preserve, + checker.source(), + checker.tokens(), + )?; Ok(Fix::unsafe_edit(edit)) }); } diff --git a/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_dict_kwargs.rs b/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_dict_kwargs.rs index f2abfc8694..1b3faba504 100644 --- a/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_dict_kwargs.rs +++ b/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_dict_kwargs.rs @@ -129,6 +129,7 @@ pub(crate) fn unnecessary_dict_kwargs(checker: &Checker, call: &ast::ExprCall) { keyword, &call.arguments, Parentheses::Preserve, + checker.source(), checker.tokens(), ) .map(Fix::safe_edit) diff --git a/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_range_start.rs b/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_range_start.rs index 692d17a55c..e12af6e069 100644 --- a/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_range_start.rs +++ b/crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_range_start.rs @@ -76,6 +76,7 @@ pub(crate) fn unnecessary_range_start(checker: &Checker, call: &ast::ExprCall) { start, &call.arguments, Parentheses::Preserve, + checker.source(), checker.tokens(), ) .map(Fix::safe_edit) diff --git a/crates/ruff_linter/src/rules/flake8_pyi/rules/generic_not_last_base_class.rs b/crates/ruff_linter/src/rules/flake8_pyi/rules/generic_not_last_base_class.rs index 88e6ef25c0..90b8476809 100644 --- a/crates/ruff_linter/src/rules/flake8_pyi/rules/generic_not_last_base_class.rs +++ b/crates/ruff_linter/src/rules/flake8_pyi/rules/generic_not_last_base_class.rs @@ -159,9 +159,16 @@ fn generate_fix( checker: &Checker, ) -> anyhow::Result { let locator = checker.locator(); + let source = locator.contents(); let tokens = checker.tokens(); - let deletion = remove_argument(generic_base, arguments, Parentheses::Preserve, tokens)?; + let deletion = remove_argument( + generic_base, + arguments, + Parentheses::Preserve, + source, + tokens, + )?; let insertion = add_argument(locator.slice(generic_base), arguments, tokens); Ok(Fix::unsafe_edits(deletion, [insertion])) diff --git a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/fixture.rs b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/fixture.rs index 6398053550..c939d44346 100644 --- a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/fixture.rs +++ b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/fixture.rs @@ -768,6 +768,7 @@ fn check_fixture_decorator(checker: &Checker, func_name: &str, decorator: &Decor keyword, arguments, edits::Parentheses::Preserve, + checker.source(), checker.tokens(), ) .map(Fix::unsafe_edit) diff --git a/crates/ruff_linter/src/rules/flake8_use_pathlib/rules/path_constructor_current_directory.rs b/crates/ruff_linter/src/rules/flake8_use_pathlib/rules/path_constructor_current_directory.rs index 6c9b48c6c6..840befca76 100644 --- a/crates/ruff_linter/src/rules/flake8_use_pathlib/rules/path_constructor_current_directory.rs +++ b/crates/ruff_linter/src/rules/flake8_use_pathlib/rules/path_constructor_current_directory.rs @@ -106,7 +106,13 @@ pub(crate) fn path_constructor_current_directory( diagnostic.set_fix(Fix::applicable_edit(edit, applicability(parent_range))); } None => diagnostic.try_set_fix(|| { - let edit = remove_argument(arg, arguments, Parentheses::Preserve, checker.tokens())?; + let edit = remove_argument( + arg, + arguments, + Parentheses::Preserve, + checker.source(), + checker.tokens(), + )?; Ok(Fix::applicable_edit(edit, applicability(call.range()))) }), } diff --git a/crates/ruff_linter/src/rules/pandas_vet/rules/inplace_argument.rs b/crates/ruff_linter/src/rules/pandas_vet/rules/inplace_argument.rs index 4029002ecd..71e0d82db9 100644 --- a/crates/ruff_linter/src/rules/pandas_vet/rules/inplace_argument.rs +++ b/crates/ruff_linter/src/rules/pandas_vet/rules/inplace_argument.rs @@ -123,8 +123,14 @@ fn convert_inplace_argument_to_assignment( ); // Remove the `inplace` argument. - let remove_argument = - remove_argument(keyword, &call.arguments, Parentheses::Preserve, tokens).ok()?; + let remove_argument = remove_argument( + keyword, + &call.arguments, + Parentheses::Preserve, + locator.contents(), + tokens, + ) + .ok()?; Some(Fix::unsafe_edits(insert_assignment, [remove_argument])) } diff --git a/crates/ruff_linter/src/rules/pylint/rules/duplicate_bases.rs b/crates/ruff_linter/src/rules/pylint/rules/duplicate_bases.rs index 7918bff89e..09ce874a09 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/duplicate_bases.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/duplicate_bases.rs @@ -94,18 +94,23 @@ pub(crate) fn duplicate_bases(checker: &Checker, name: &str, arguments: Option<& base.range(), ); diagnostic.try_set_fix(|| { - remove_argument(base, arguments, Parentheses::Remove, checker.tokens()).map( - |edit| { - Fix::applicable_edit( - edit, - if checker.comment_ranges().intersects(arguments.range()) { - Applicability::Unsafe - } else { - Applicability::Safe - }, - ) - }, + remove_argument( + base, + arguments, + Parentheses::Remove, + checker.locator().contents(), + checker.tokens(), ) + .map(|edit| { + Fix::applicable_edit( + edit, + if checker.comment_ranges().intersects(arguments.range()) { + Applicability::Unsafe + } else { + Applicability::Safe + }, + ) + }) }); } } diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/pep695/non_pep695_generic_class.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/pep695/non_pep695_generic_class.rs index d97fb684e1..02f8cdd1e8 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/pep695/non_pep695_generic_class.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/pep695/non_pep695_generic_class.rs @@ -203,6 +203,7 @@ pub(crate) fn non_pep695_generic_class(checker: &Checker, class_def: &StmtClassD generic_expr, arguments, Parentheses::Remove, + checker.source(), checker.tokens(), )?; Ok(Fix::unsafe_edits( diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/replace_stdout_stderr.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/replace_stdout_stderr.rs index 6f3e5256df..6ce403647d 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/replace_stdout_stderr.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/replace_stdout_stderr.rs @@ -97,7 +97,15 @@ pub(crate) fn replace_stdout_stderr(checker: &Checker, call: &ast::ExprCall) { let mut diagnostic = checker.report_diagnostic(ReplaceStdoutStderr, call.range()); if call.arguments.find_keyword("capture_output").is_none() { - diagnostic.try_set_fix(|| generate_fix(stdout, stderr, call, checker.tokens())); + diagnostic.try_set_fix(|| { + generate_fix( + stdout, + stderr, + call, + checker.locator().contents(), + checker.tokens(), + ) + }); } } } @@ -107,6 +115,7 @@ fn generate_fix( stdout: &Keyword, stderr: &Keyword, call: &ast::ExprCall, + source: &str, tokens: &Tokens, ) -> Result { let (first, second) = if stdout.start() < stderr.start() { @@ -121,6 +130,7 @@ fn generate_fix( second, &call.arguments, Parentheses::Preserve, + source, tokens, )?], )) diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/replace_universal_newlines.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/replace_universal_newlines.rs index c4a28c52ba..b7b13c7b66 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/replace_universal_newlines.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/replace_universal_newlines.rs @@ -77,6 +77,7 @@ pub(crate) fn replace_universal_newlines(checker: &Checker, call: &ast::ExprCall kwarg, &call.arguments, Parentheses::Preserve, + checker.locator().contents(), checker.tokens(), ) .map(Fix::safe_edit) diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs index 2583849bbc..792365042f 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs @@ -187,6 +187,7 @@ pub(crate) fn unnecessary_encode_utf8(checker: &Checker, call: &ast::ExprCall) { kwarg, &call.arguments, Parentheses::Preserve, + checker.locator().contents(), checker.tokens(), ) .map(Fix::safe_edit) @@ -204,6 +205,7 @@ pub(crate) fn unnecessary_encode_utf8(checker: &Checker, call: &ast::ExprCall) { arg, &call.arguments, Parentheses::Preserve, + checker.locator().contents(), checker.tokens(), ) .map(Fix::safe_edit) @@ -228,6 +230,7 @@ pub(crate) fn unnecessary_encode_utf8(checker: &Checker, call: &ast::ExprCall) { kwarg, &call.arguments, Parentheses::Preserve, + checker.locator().contents(), checker.tokens(), ) .map(Fix::safe_edit) @@ -245,6 +248,7 @@ pub(crate) fn unnecessary_encode_utf8(checker: &Checker, call: &ast::ExprCall) { arg, &call.arguments, Parentheses::Preserve, + checker.locator().contents(), checker.tokens(), ) .map(Fix::safe_edit) diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/useless_class_metaclass_type.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/useless_class_metaclass_type.rs index fa9b9910f1..20d3f64461 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/useless_class_metaclass_type.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/useless_class_metaclass_type.rs @@ -65,8 +65,13 @@ pub(crate) fn useless_class_metaclass_type(checker: &Checker, class_def: &StmtCl ); diagnostic.try_set_fix(|| { - let edit = - remove_argument(keyword, arguments, Parentheses::Remove, checker.tokens())?; + let edit = remove_argument( + keyword, + arguments, + Parentheses::Remove, + checker.locator().contents(), + checker.tokens(), + )?; let range = edit.range(); let applicability = if checker.comment_ranges().intersects(range) { diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/useless_object_inheritance.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/useless_object_inheritance.rs index c686fc57bf..a1b0d900f8 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/useless_object_inheritance.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/useless_object_inheritance.rs @@ -68,7 +68,13 @@ pub(crate) fn useless_object_inheritance(checker: &Checker, class_def: &ast::Stm ); diagnostic.try_set_fix(|| { - let edit = remove_argument(base, arguments, Parentheses::Remove, checker.tokens())?; + let edit = remove_argument( + base, + arguments, + Parentheses::Remove, + checker.locator().contents(), + checker.tokens(), + )?; let range = edit.range(); let applicability = if checker.comment_ranges().intersects(range) { diff --git a/crates/ruff_linter/src/rules/ruff/rules/class_with_mixed_type_vars.rs b/crates/ruff_linter/src/rules/ruff/rules/class_with_mixed_type_vars.rs index 31fa53e34a..7b023de830 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/class_with_mixed_type_vars.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/class_with_mixed_type_vars.rs @@ -162,6 +162,7 @@ fn convert_type_vars( generic_base, class_arguments, Parentheses::Remove, + source, checker.tokens(), )?; let replace_type_params = diff --git a/crates/ruff_linter/src/rules/ruff/rules/default_factory_kwarg.rs b/crates/ruff_linter/src/rules/ruff/rules/default_factory_kwarg.rs index 44104034be..70ef8cb6d4 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/default_factory_kwarg.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/default_factory_kwarg.rs @@ -151,6 +151,7 @@ fn convert_to_positional( default_factory, &call.arguments, Parentheses::Preserve, + locator.contents(), tokens, )?; diff --git a/crates/ruff_linter/src/rules/ruff/rules/falsy_dict_get_fallback.rs b/crates/ruff_linter/src/rules/ruff/rules/falsy_dict_get_fallback.rs index 6a80acb24d..de3c072bdd 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/falsy_dict_get_fallback.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/falsy_dict_get_fallback.rs @@ -127,6 +127,7 @@ pub(crate) fn falsy_dict_get_fallback(checker: &Checker, expr: &Expr) { &fallback_arg, &call.arguments, Parentheses::Preserve, + checker.locator().contents(), checker.tokens(), ) .map(|edit| Fix::applicable_edit(edit, applicability)) diff --git a/crates/ruff_linter/src/rules/ruff/rules/unnecessary_literal_within_deque_call.rs b/crates/ruff_linter/src/rules/ruff/rules/unnecessary_literal_within_deque_call.rs index fcf1f395df..b322d57fd6 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/unnecessary_literal_within_deque_call.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/unnecessary_literal_within_deque_call.rs @@ -151,6 +151,7 @@ fn fix_unnecessary_literal_in_deque( &iterable, &deque.arguments, Parentheses::Preserve, + checker.source(), checker.tokens(), )? };