Python
/
ruff
mirror of https://github.com/astral-sh/ruff
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use shared finder

This commit is contained in:
Charlie Marsh 2024-02-08 18:05:51 -05:00
parent 56b148bb43
commit c67d68271d
3 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -2343,6 +2343,7 @@ dependencies = [
"lalrpop", "lalrpop",
"lalrpop-util", "lalrpop-util",
"memchr", "memchr",
"once_cell",
"ruff_python_ast", "ruff_python_ast",
"ruff_text_size", "ruff_text_size",
"rustc-hash", "rustc-hash",

1
crates/ruff_python_parser/Cargo.toml
View File

@ -28,6 +28,7 @@ rustc-hash = { workspace = true }
static_assertions = { workspace = true } static_assertions = { workspace = true }
unicode-ident = { workspace = true } unicode-ident = { workspace = true }
unicode_names2 = { workspace = true } unicode_names2 = { workspace = true }
once_cell = "1.19.0"
[dev-dependencies] [dev-dependencies]
insta = { workspace = true } insta = { workspace = true }

12
crates/ruff_python_parser/src/string.rs
View File

@ -1,6 +1,8 @@
//! Parsing of string literals, bytes literals, and implicit string concatenation. //! Parsing of string literals, bytes literals, and implicit string concatenation.
use bstr::ByteSlice; use bstr::ByteSlice;
use memchr::memmem;
use once_cell::sync::Lazy;
use ruff_python_ast::{self as ast, Expr}; use ruff_python_ast::{self as ast, Expr};
use ruff_text_size::{Ranged, TextRange, TextSize}; use ruff_text_size::{Ranged, TextRange, TextSize};
@ -8,6 +10,8 @@ use ruff_text_size::{Ranged, TextRange, TextSize};
use crate::lexer::{LexicalError, LexicalErrorType}; use crate::lexer::{LexicalError, LexicalErrorType};
use crate::token::{StringKind, Tok}; use crate::token::{StringKind, Tok};
const BACKSLASH_FINDER: Lazy<memmem::Finder> = Lazy::new(|| memmem::Finder::new(b"\\"));
pub(crate) enum StringType { pub(crate) enum StringType {
Str(ast::StringLiteral), Str(ast::StringLiteral),
Bytes(ast::BytesLiteral), Bytes(ast::BytesLiteral),
@ -310,7 +314,7 @@ impl StringParser {
})); }));
} }
let Some(mut escape) = memchr::memchr(b'\\', self.source.as_bytes()) else { let Some(mut escape) = BACKSLASH_FINDER.find(self.source.as_bytes()) else {
// If the string doesn't contain any escape sequences, return the owned string. // If the string doesn't contain any escape sequences, return the owned string.
return Ok(StringType::Bytes(ast::BytesLiteral { return Ok(StringType::Bytes(ast::BytesLiteral {
value: self.source.into_boxed_bytes(), value: self.source.into_boxed_bytes(),
@ -336,7 +340,7 @@ impl StringParser {
} }
} }
let Some(next_escape) = memchr::memchr(b'\\', self.source[self.cursor..].as_bytes()) let Some(next_escape) = BACKSLASH_FINDER.find(self.source[self.cursor..].as_bytes())
else { else {
// Add the rest of the string to the value. // Add the rest of the string to the value.
let rest = &self.source[self.cursor..]; let rest = &self.source[self.cursor..];
@ -364,7 +368,7 @@ impl StringParser {
})); }));
} }
let Some(mut escape) = memchr::memchr(b'\\', self.source.as_bytes()) else { let Some(mut escape) = BACKSLASH_FINDER.find(self.source.as_bytes()) else {
// If the string doesn't contain any escape sequences, return the owned string. // If the string doesn't contain any escape sequences, return the owned string.
return Ok(StringType::Str(ast::StringLiteral { return Ok(StringType::Str(ast::StringLiteral {
value: self.source, value: self.source,
@ -392,7 +396,7 @@ impl StringParser {
} }
} }
let Some(next_escape) = memchr::memchr(b'\\', self.source[self.cursor..].as_bytes()) let Some(next_escape) = BACKSLASH_FINDER.find(self.source[self.cursor..].as_bytes())
else { else {
// Add the rest of the string to the value. // Add the rest of the string to the value.
let rest = &self.source[self.cursor..]; let rest = &self.source[self.cursor..];