Removed unicode literals (#1448)

README.md

@@ -678,6 +678,7 @@ For more, see [pyupgrade](https://pypi.org/project/pyupgrade/3.2.0/) on PyPI.
 | UP021 | ReplaceUniversalNewlines | `universal_newlines` is deprecated, use `text` | 🛠 |
 | UP022 | ReplaceStdoutStderr | Sending stdout and stderr to pipe is deprecated, use `capture_output` | 🛠 |
 | UP023 | RewriteCElementTree | `cElementTree` is deprecated, use `ElementTree` | 🛠 |
+| UP025 | RewriteUnicodeLiteral | Remove unicode literals from strings | 🛠 |
 
 ### pep8-naming (N)
 

resources/test/fixtures/pyupgrade/UP025.py

@@ -0,0 +1,27 @@
+# These should change
+x = u"Hello"
+
+u'world'
+
+print(u"Hello")
+
+print(u'world')
+
+import foo
+
+foo(u"Hello", U"world", a=u"Hello", b=u"world")
+
+# These should stay quoted they way they are
+
+x = u'hello'
+x = u"""hello"""
+x = u'''hello'''
+x = u'Hello "World"'
+
+# These should not change
+u = "Hello"
+
+u = u
+
+def hello():
+    return"Hello"

ruff.schema.json

@@ -891,6 +891,7 @@
         "UP021",
         "UP022",
         "UP023",
+        "UP025",
         "W",
         "W2",
         "W29",

src/checkers/ast.rs

@@ -2329,7 +2329,7 @@ where
             }
             ExprKind::Constant {
                 value: Constant::Str(value),
-                ..
+                kind,
             } => {
                 if self.in_type_definition && !self.in_literal {
                     self.deferred_string_type_definitions.push((
@@ -2347,6 +2347,9 @@ where
                         self.add_check(check);
                     }
                 }
+                if self.settings.enabled.contains(&CheckCode::UP025) {
+                    pyupgrade::plugins::rewrite_unicode_literal(self, expr, value, kind);
+                }
             }
             ExprKind::Lambda { args, body, .. } => {
                 // Visit the arguments, but avoid the body, which will be deferred.

src/checks.rs

@@ -231,6 +231,7 @@ pub enum CheckCode {
     UP021,
     UP022,
     UP023,
+    UP025,
     // pydocstyle
     D100,
     D101,
@@ -849,6 +850,7 @@ pub enum CheckKind {
     ReplaceUniversalNewlines,
     ReplaceStdoutStderr,
     RewriteCElementTree,
+    RewriteUnicodeLiteral,
     // pydocstyle
     BlankLineAfterLastSection(String),
     BlankLineAfterSection(String),
@@ -1233,6 +1235,7 @@ impl CheckCode {
             CheckCode::UP021 => CheckKind::ReplaceUniversalNewlines,
             CheckCode::UP022 => CheckKind::ReplaceStdoutStderr,
             CheckCode::UP023 => CheckKind::RewriteCElementTree,
+            CheckCode::UP025 => CheckKind::RewriteUnicodeLiteral,
             // pydocstyle
             CheckCode::D100 => CheckKind::PublicModule,
             CheckCode::D101 => CheckKind::PublicClass,
@@ -1661,6 +1664,7 @@ impl CheckCode {
             CheckCode::UP021 => CheckCategory::Pyupgrade,
             CheckCode::UP022 => CheckCategory::Pyupgrade,
             CheckCode::UP023 => CheckCategory::Pyupgrade,
+            CheckCode::UP025 => CheckCategory::Pyupgrade,
             CheckCode::W292 => CheckCategory::Pycodestyle,
             CheckCode::W605 => CheckCategory::Pycodestyle,
             CheckCode::YTT101 => CheckCategory::Flake82020,
@@ -1878,6 +1882,7 @@ impl CheckKind {
             CheckKind::ReplaceUniversalNewlines => &CheckCode::UP021,
             CheckKind::ReplaceStdoutStderr => &CheckCode::UP022,
             CheckKind::RewriteCElementTree => &CheckCode::UP023,
+            CheckKind::RewriteUnicodeLiteral => &CheckCode::UP025,
             // pydocstyle
             CheckKind::BlankLineAfterLastSection(..) => &CheckCode::D413,
             CheckKind::BlankLineAfterSection(..) => &CheckCode::D410,
@@ -2618,6 +2623,7 @@ impl CheckKind {
             CheckKind::RewriteCElementTree => {
                 "`cElementTree` is deprecated, use `ElementTree`".to_string()
             }
+            CheckKind::RewriteUnicodeLiteral => "Remove unicode literals from strings".to_string(),
             CheckKind::ConvertNamedTupleFunctionalToClass(name) => {
                 format!("Convert `{name}` from `NamedTuple` functional to class syntax")
             }
@@ -3066,6 +3072,7 @@ impl CheckKind {
                 | CheckKind::ReplaceUniversalNewlines
                 | CheckKind::ReplaceStdoutStderr
                 | CheckKind::RewriteCElementTree
+                | CheckKind::RewriteUnicodeLiteral
                 | CheckKind::NewLineAfterSectionName(..)
                 | CheckKind::NoBlankLineAfterFunction(..)
                 | CheckKind::NoBlankLineBeforeClass(..)

src/checks_gen.rs

@@ -535,6 +535,7 @@ pub enum CheckCodePrefix {
     UP021,
     UP022,
     UP023,
+    UP025,
     W,
     W2,
     W29,
@@ -764,6 +765,7 @@ impl CheckCodePrefix {
                 CheckCode::UP021,
                 CheckCode::UP022,
                 CheckCode::UP023,
+                CheckCode::UP025,
                 CheckCode::D100,
                 CheckCode::D101,
                 CheckCode::D102,
@@ -2439,6 +2441,7 @@ impl CheckCodePrefix {
                     CheckCode::UP021,
                     CheckCode::UP022,
                     CheckCode::UP023,
+                    CheckCode::UP025,
                 ]
             }
             CheckCodePrefix::U0 => {
@@ -2471,6 +2474,7 @@ impl CheckCodePrefix {
                     CheckCode::UP021,
                     CheckCode::UP022,
                     CheckCode::UP023,
+                    CheckCode::UP025,
                 ]
             }
             CheckCodePrefix::U00 => {
@@ -2687,6 +2691,7 @@ impl CheckCodePrefix {
                 CheckCode::UP021,
                 CheckCode::UP022,
                 CheckCode::UP023,
+                CheckCode::UP025,
             ],
             CheckCodePrefix::UP0 => vec![
                 CheckCode::UP001,
@@ -2711,6 +2716,7 @@ impl CheckCodePrefix {
                 CheckCode::UP021,
                 CheckCode::UP022,
                 CheckCode::UP023,
+                CheckCode::UP025,
             ],
             CheckCodePrefix::UP00 => vec![
                 CheckCode::UP001,
@@ -2757,11 +2763,13 @@ impl CheckCodePrefix {
                 CheckCode::UP021,
                 CheckCode::UP022,
                 CheckCode::UP023,
+                CheckCode::UP025,
             ],
             CheckCodePrefix::UP020 => vec![CheckCode::UP020],
             CheckCodePrefix::UP021 => vec![CheckCode::UP021],
             CheckCodePrefix::UP022 => vec![CheckCode::UP022],
             CheckCodePrefix::UP023 => vec![CheckCode::UP023],
+            CheckCodePrefix::UP025 => vec![CheckCode::UP025],
             CheckCodePrefix::W => vec![CheckCode::W292, CheckCode::W605],
             CheckCodePrefix::W2 => vec![CheckCode::W292],
             CheckCodePrefix::W29 => vec![CheckCode::W292],
@@ -3328,6 +3336,7 @@ impl CheckCodePrefix {
             CheckCodePrefix::UP021 => SuffixLength::Three,
             CheckCodePrefix::UP022 => SuffixLength::Three,
             CheckCodePrefix::UP023 => SuffixLength::Three,
+            CheckCodePrefix::UP025 => SuffixLength::Three,
             CheckCodePrefix::W => SuffixLength::Zero,
             CheckCodePrefix::W2 => SuffixLength::One,
             CheckCodePrefix::W29 => SuffixLength::Two,

src/pyupgrade/mod.rs

@@ -42,6 +42,7 @@ mod tests {
     #[test_case(CheckCode::UP021, Path::new("UP021.py"); "UP021")]
     #[test_case(CheckCode::UP022, Path::new("UP022.py"); "UP022")]
     #[test_case(CheckCode::UP023, Path::new("UP023.py"); "UP023")]
+    #[test_case(CheckCode::UP025, Path::new("UP025.py"); "UP025")]
     fn checks(check_code: CheckCode, path: &Path) -> Result<()> {
         let snapshot = format!("{}_{}", check_code.as_ref(), path.to_string_lossy());
         let mut checks = test_path(

src/pyupgrade/plugins/mod.rs

@@ -9,6 +9,7 @@ pub use remove_six_compat::remove_six_compat;
 pub use replace_stdout_stderr::replace_stdout_stderr;
 pub use replace_universal_newlines::replace_universal_newlines;
 pub use rewrite_c_element_tree::replace_c_element_tree;
+pub use rewrite_unicode_literal::rewrite_unicode_literal;
 pub use super_call_with_parameters::super_call_with_parameters;
 pub use type_of_primitive::type_of_primitive;
 pub use typing_text_str_alias::typing_text_str_alias;
@@ -31,6 +32,7 @@ mod remove_six_compat;
 mod replace_stdout_stderr;
 mod replace_universal_newlines;
 mod rewrite_c_element_tree;
+mod rewrite_unicode_literal;
 mod super_call_with_parameters;
 mod type_of_primitive;
 mod typing_text_str_alias;

src/pyupgrade/plugins/rewrite_unicode_literal.rs

@@ -0,0 +1,48 @@
+use rustpython_ast::Expr;
+
+use crate::ast::types::Range;
+use crate::autofix::Fix;
+use crate::checkers::ast::Checker;
+use crate::checks::{Check, CheckKind};
+use crate::pydocstyle::helpers::leading_quote;
+
+/// Strip any leading kind prefixes (e..g. "u") from a quote string.
+fn strip_kind(leading_quote: &str) -> &str {
+    if let Some(index) = leading_quote.find('\'') {
+        &leading_quote[index..]
+    } else if let Some(index) = leading_quote.find('\"') {
+        &leading_quote[index..]
+    } else {
+        unreachable!("Expected docstring to start with a valid triple- or single-quote prefix")
+    }
+}
+
+pub fn rewrite_unicode_literal(
+    checker: &mut Checker,
+    expr: &Expr,
+    value: &str,
+    kind: &Option<String>,
+) {
+    if let Some(const_kind) = kind {
+        if const_kind.to_lowercase() == "u" {
+            let mut check = Check::new(CheckKind::RewriteUnicodeLiteral, Range::from_located(expr));
+            if checker.patch(check.kind.code()) {
+                let content = checker
+                    .locator
+                    .slice_source_code_range(&Range::from_located(expr));
+                if let Some(leading_quote) = leading_quote(&content).map(strip_kind) {
+                    let mut contents = String::with_capacity(value.len() + leading_quote.len() * 2);
+                    contents.push_str(leading_quote);
+                    contents.push_str(value);
+                    contents.push_str(leading_quote);
+                    check.amend(Fix::replacement(
+                        contents,
+                        expr.location,
+                        expr.end_location.unwrap(),
+                    ));
+                }
+            }
+            checker.add_check(check);
+        }
+    }
+}

src/pyupgrade/snapshots/ruff__pyupgrade__tests__UP025_UP025.py.snap

@@ -0,0 +1,185 @@
+---
+source: src/pyupgrade/mod.rs
+expression: checks
+---
+- kind: RewriteUnicodeLiteral
+  location:
+    row: 2
+    column: 4
+  end_location:
+    row: 2
+    column: 12
+  fix:
+    content: "\"Hello\""
+    location:
+      row: 2
+      column: 4
+    end_location:
+      row: 2
+      column: 12
+- kind: RewriteUnicodeLiteral
+  location:
+    row: 4
+    column: 0
+  end_location:
+    row: 4
+    column: 8
+  fix:
+    content: "'world'"
+    location:
+      row: 4
+      column: 0
+    end_location:
+      row: 4
+      column: 8
+- kind: RewriteUnicodeLiteral
+  location:
+    row: 6
+    column: 6
+  end_location:
+    row: 6
+    column: 14
+  fix:
+    content: "\"Hello\""
+    location:
+      row: 6
+      column: 6
+    end_location:
+      row: 6
+      column: 14
+- kind: RewriteUnicodeLiteral
+  location:
+    row: 8
+    column: 6
+  end_location:
+    row: 8
+    column: 14
+  fix:
+    content: "'world'"
+    location:
+      row: 8
+      column: 6
+    end_location:
+      row: 8
+      column: 14
+- kind: RewriteUnicodeLiteral
+  location:
+    row: 12
+    column: 4
+  end_location:
+    row: 12
+    column: 12
+  fix:
+    content: "\"Hello\""
+    location:
+      row: 12
+      column: 4
+    end_location:
+      row: 12
+      column: 12
+- kind: RewriteUnicodeLiteral
+  location:
+    row: 12
+    column: 14
+  end_location:
+    row: 12
+    column: 22
+  fix:
+    content: "\"world\""
+    location:
+      row: 12
+      column: 14
+    end_location:
+      row: 12
+      column: 22
+- kind: RewriteUnicodeLiteral
+  location:
+    row: 12
+    column: 26
+  end_location:
+    row: 12
+    column: 34
+  fix:
+    content: "\"Hello\""
+    location:
+      row: 12
+      column: 26
+    end_location:
+      row: 12
+      column: 34
+- kind: RewriteUnicodeLiteral
+  location:
+    row: 12
+    column: 38
+  end_location:
+    row: 12
+    column: 46
+  fix:
+    content: "\"world\""
+    location:
+      row: 12
+      column: 38
+    end_location:
+      row: 12
+      column: 46
+- kind: RewriteUnicodeLiteral
+  location:
+    row: 16
+    column: 4
+  end_location:
+    row: 16
+    column: 12
+  fix:
+    content: "'hello'"
+    location:
+      row: 16
+      column: 4
+    end_location:
+      row: 16
+      column: 12
+- kind: RewriteUnicodeLiteral
+  location:
+    row: 17
+    column: 4
+  end_location:
+    row: 17
+    column: 16
+  fix:
+    content: "\"\"\"hello\"\"\""
+    location:
+      row: 17
+      column: 4
+    end_location:
+      row: 17
+      column: 16
+- kind: RewriteUnicodeLiteral
+  location:
+    row: 18
+    column: 4
+  end_location:
+    row: 18
+    column: 16
+  fix:
+    content: "'''hello'''"
+    location:
+      row: 18
+      column: 4
+    end_location:
+      row: 18
+      column: 16
+- kind: RewriteUnicodeLiteral
+  location:
+    row: 19
+    column: 4
+  end_location:
+    row: 19
+    column: 20
+  fix:
+    content: "'Hello \"World\"'"
+    location:
+      row: 19
+      column: 4
+    end_location:
+      row: 19
+      column: 20
+