Python
/
ruff
mirror of https://github.com/astral-sh/ruff
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix clippy issues

This commit is contained in:
Micha Reiser 2023-07-27 11:31:10 +02:00
parent 6f4ee32807
commit 802df97d55
No known key found for this signature in database
17 changed files with 657 additions and 629 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
rustpython_original/literal/src/cformat.rs
View File

@ -32,7 +32,9 @@ pub struct CFormatError {
impl fmt::Display for CFormatError { impl fmt::Display for CFormatError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
use CFormatErrorType::*; use CFormatErrorType::{
IncompleteFormat, IntTooBig, UnmatchedKeyParentheses, UnsupportedFormatChar,
};
match self.typ { match self.typ {
UnmatchedKeyParentheses => write!(f, "incomplete format key"), UnmatchedKeyParentheses => write!(f, "incomplete format key"),
IncompleteFormat => write!(f, "incomplete format"), IncompleteFormat => write!(f, "incomplete format"),
@ -42,7 +44,9 @@ impl fmt::Display for CFormatError {
c, c as u32, self.index c, c as u32, self.index
), ),
IntTooBig => write!(f, "width/precision too big"), IntTooBig => write!(f, "width/precision too big"),
_ => write!(f, "unexpected error parsing format string"), CFormatErrorType::MissingModuloSign => {
write!(f, "unexpected error parsing format string")
}
} }
} }
} }
@ -187,14 +191,14 @@ impl CFormatSpec {
let fill_chars_needed = width.saturating_sub(num_chars); let fill_chars_needed = width.saturating_sub(num_chars);
let fill_string = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed); let fill_string = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed);
if !fill_string.is_empty() { if fill_string.is_empty() {
string
} else {
if self.flags.contains(CConversionFlags::LEFT_ADJUST) { if self.flags.contains(CConversionFlags::LEFT_ADJUST) {
format!("{string}{fill_string}") format!("{string}{fill_string}")
} else { } else {
format!("{fill_string}{string}") format!("{fill_string}{string}")
} }
} else {
string
} }
} }
@ -210,13 +214,13 @@ impl CFormatSpec {
let fill_chars_needed = width.saturating_sub(num_chars); let fill_chars_needed = width.saturating_sub(num_chars);
let fill_string = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed); let fill_string = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed);
if !fill_string.is_empty() { if fill_string.is_empty() {
string
} else {
// Don't left-adjust if precision-filling: that will always be prepending 0s to %d // Don't left-adjust if precision-filling: that will always be prepending 0s to %d
// arguments, the LEFT_ADJUST flag will be used by a later call to fill_string with // arguments, the LEFT_ADJUST flag will be used by a later call to fill_string with
// the 0-filled string as the string param. // the 0-filled string as the string param.
format!("{fill_string}{string}") format!("{fill_string}{string}")
} else {
string
} }
} }
@ -279,7 +283,7 @@ impl CFormatSpec {
} }
pub fn format_number(&self, num: &BigInt) -> String { pub fn format_number(&self, num: &BigInt) -> String {
use CNumberType::*; use CNumberType::{Decimal, Hex, Octal};
let magnitude = num.abs(); let magnitude = num.abs();
let prefix = if self.flags.contains(CConversionFlags::ALTERNATE_FORM) { let prefix = if self.flags.contains(CConversionFlags::ALTERNATE_FORM) {
match self.format_type { match self.format_type {
@ -312,10 +316,10 @@ impl CFormatSpec {
let padded_magnitude_string = self.fill_string_with_precision(magnitude_string, '0'); let padded_magnitude_string = self.fill_string_with_precision(magnitude_string, '0');
if self.flags.contains(CConversionFlags::ZERO_PAD) { if self.flags.contains(CConversionFlags::ZERO_PAD) {
let fill_char = if !self.flags.contains(CConversionFlags::LEFT_ADJUST) { let fill_char = if self.flags.contains(CConversionFlags::LEFT_ADJUST) {
'0'
} else {
' ' // '-' overrides the '0' conversion if both are given ' ' // '-' overrides the '0' conversion if both are given
} else {
'0'
}; };
let signed_prefix = format!("{sign_string}{prefix}"); let signed_prefix = format!("{sign_string}{prefix}");
format!( format!(
@ -386,10 +390,10 @@ impl CFormatSpec {
}; };
if self.flags.contains(CConversionFlags::ZERO_PAD) { if self.flags.contains(CConversionFlags::ZERO_PAD) {
let fill_char = if !self.flags.contains(CConversionFlags::LEFT_ADJUST) { let fill_char = if self.flags.contains(CConversionFlags::LEFT_ADJUST) {
'0'
} else {
' ' ' '
} else {
'0'
}; };
format!( format!(
"{}{}", "{}{}",
@ -462,14 +466,14 @@ where
T: Into<char>, T: Into<char>,
I: Iterator<Item = T>, I: Iterator<Item = T>,
{ {
use CFloatType::*; use CFloatType::{Exponent, General, PointDecimal};
use CNumberType::*; use CNumberType::{Decimal, Hex, Octal};
let (index, c) = match iter.next() { let (index, c) = match iter.next() {
Some((index, c)) => (index, c.into()), Some((index, c)) => (index, c.into()),
None => { None => {
return Err(( return Err((
CFormatErrorType::IncompleteFormat, CFormatErrorType::IncompleteFormat,
iter.peek().map(|x| x.0).unwrap_or(0), iter.peek().map_or(0, |x| x.0),
)); ));
} }
}; };
@ -494,6 +498,7 @@ where
Ok((format_type, c)) Ok((format_type, c))
} }
#[allow(clippy::cast_possible_wrap)]
fn parse_quantity<T, I>(iter: &mut ParseIter<I>) -> Result<Option<CFormatQuantity>, ParsingError> fn parse_quantity<T, I>(iter: &mut ParseIter<I>) -> Result<Option<CFormatQuantity>, ParsingError>
where where
T: Into<char> + Copy, T: Into<char> + Copy,
@ -587,7 +592,7 @@ impl<T> CFormatPart<T> {
pub fn has_key(&self) -> bool { pub fn has_key(&self) -> bool {
match self { match self {
CFormatPart::Spec(s) => s.mapping_key.is_some(), CFormatPart::Spec(s) => s.mapping_key.is_some(),
_ => false, CFormatPart::Literal(_) => false,
} }
} }
} }
@ -640,21 +645,20 @@ impl CFormatBytes {
iter.next().unwrap(); iter.next().unwrap();
literal.push(b'%'); literal.push(b'%');
continue; continue;
} else { }
if !literal.is_empty() { if !literal.is_empty() {
parts.push(( parts.push((
part_index, part_index,
CFormatPart::Literal(std::mem::take(&mut literal)), CFormatPart::Literal(std::mem::take(&mut literal)),
)); ));
} }
let spec = CFormatSpec::parse(iter).map_err(|err| CFormatError { let spec = CFormatSpec::parse(iter).map_err(|err| CFormatError {
typ: err.0, typ: err.0,
index: err.1, index: err.1,
})?; })?;
parts.push((index, CFormatPart::Spec(spec))); parts.push((index, CFormatPart::Spec(spec)));
if let Some(&(index, _)) = iter.peek() { if let Some(&(index, _)) = iter.peek() {
part_index = index; part_index = index;
}
} }
} else { } else {
return Err(CFormatError { return Err(CFormatError {
@ -673,7 +677,7 @@ impl CFormatBytes {
} }
pub fn parse_from_bytes(bytes: &[u8]) -> Result<Self, CFormatError> { pub fn parse_from_bytes(bytes: &[u8]) -> Result<Self, CFormatError> {
let mut iter = bytes.iter().cloned().enumerate().peekable(); let mut iter = bytes.iter().copied().enumerate().peekable();
Self::parse(&mut iter) Self::parse(&mut iter)
} }
} }
@ -701,21 +705,20 @@ impl CFormatString {
iter.next().unwrap(); iter.next().unwrap();
literal.push('%'); literal.push('%');
continue; continue;
} else { }
if !literal.is_empty() { if !literal.is_empty() {
parts.push(( parts.push((
part_index, part_index,
CFormatPart::Literal(std::mem::take(&mut literal)), CFormatPart::Literal(std::mem::take(&mut literal)),
)); ));
} }
let spec = CFormatSpec::parse(iter).map_err(|err| CFormatError { let spec = CFormatSpec::parse(iter).map_err(|err| CFormatError {
typ: err.0, typ: err.0,
index: err.1, index: err.1,
})?; })?;
parts.push((index, CFormatPart::Spec(spec))); parts.push((index, CFormatPart::Spec(spec)));
if let Some(&(index, _)) = iter.peek() { if let Some(&(index, _)) = iter.peek() {
part_index = index; part_index = index;
}
} }
} else { } else {
return Err(CFormatError { return Err(CFormatError {
@ -868,7 +871,7 @@ mod tests {
.parse::<CFormatSpec>() .parse::<CFormatSpec>()
.unwrap() .unwrap()
.format_string("Hello, World!".to_owned()), .format_string("Hello, World!".to_owned()),
"".to_owned() String::new()
); );
assert_eq!( assert_eq!(
"%5.s" "%5.s"
@ -997,7 +1000,7 @@ mod tests {
assert_eq!( assert_eq!(
"%f".parse::<CFormatSpec>() "%f".parse::<CFormatSpec>()
.unwrap() .unwrap()
.format_float(1.2345678901), .format_float(1.234_567_890_1),
"1.234568" "1.234568"
); );
} }

40
rustpython_original/literal/src/escape.rs
View File

@ -6,7 +6,8 @@ pub enum Quote {
impl Quote { impl Quote {
#[inline] #[inline]
pub const fn swap(self) -> Quote { #[must_use]
pub const fn swap(self) -> Self {
match self { match self {
Quote::Single => Quote::Double, Quote::Single => Quote::Double,
Quote::Double => Quote::Single, Quote::Double => Quote::Single,
@ -126,6 +127,11 @@ impl std::fmt::Display for StrRepr<'_, '_> {
impl UnicodeEscape<'_> { impl UnicodeEscape<'_> {
const REPR_RESERVED_LEN: usize = 2; // for quotes const REPR_RESERVED_LEN: usize = 2; // for quotes
#[allow(
clippy::cast_possible_wrap,
clippy::cast_possible_truncation,
clippy::cast_sign_loss
)]
pub fn repr_layout(source: &str, preferred_quote: Quote) -> EscapeLayout { pub fn repr_layout(source: &str, preferred_quote: Quote) -> EscapeLayout {
Self::output_layout_with_checker(source, preferred_quote, |a, b| { Self::output_layout_with_checker(source, preferred_quote, |a, b| {
Some((a as isize).checked_add(b as isize)? as usize) Some((a as isize).checked_add(b as isize)? as usize)
@ -155,8 +161,15 @@ impl UnicodeEscape<'_> {
}; };
let Some(new_len) = length_add(out_len, incr) else { let Some(new_len) = length_add(out_len, incr) else {
#[cold] #[cold]
fn stop(single_count: usize, double_count: usize, preferred_quote: Quote) -> EscapeLayout { fn stop(
EscapeLayout { quote: choose_quote(single_count, double_count, preferred_quote).0, len: None } single_count: usize,
double_count: usize,
preferred_quote: Quote,
) -> EscapeLayout {
EscapeLayout {
quote: choose_quote(single_count, double_count, preferred_quote).0,
len: None,
}
} }
return stop(single_count, double_count, preferred_quote); return stop(single_count, double_count, preferred_quote);
}; };
@ -296,12 +309,22 @@ impl<'a> AsciiEscape<'a> {
} }
impl AsciiEscape<'_> { impl AsciiEscape<'_> {
#[allow(
clippy::cast_possible_wrap,
clippy::cast_possible_truncation,
clippy::cast_sign_loss
)]
pub fn repr_layout(source: &[u8], preferred_quote: Quote) -> EscapeLayout { pub fn repr_layout(source: &[u8], preferred_quote: Quote) -> EscapeLayout {
Self::output_layout_with_checker(source, preferred_quote, 3, |a, b| { Self::output_layout_with_checker(source, preferred_quote, 3, |a, b| {
Some((a as isize).checked_add(b as isize)? as usize) Some((a as isize).checked_add(b as isize)? as usize)
}) })
} }
#[allow(
clippy::cast_possible_wrap,
clippy::cast_possible_truncation,
clippy::cast_sign_loss
)]
pub fn named_repr_layout(source: &[u8], name: &str) -> EscapeLayout { pub fn named_repr_layout(source: &[u8], name: &str) -> EscapeLayout {
Self::output_layout_with_checker(source, Quote::Single, name.len() + 2 + 3, |a, b| { Self::output_layout_with_checker(source, Quote::Single, name.len() + 2 + 3, |a, b| {
Some((a as isize).checked_add(b as isize)? as usize) Some((a as isize).checked_add(b as isize)? as usize)
@ -332,8 +355,15 @@ impl AsciiEscape<'_> {
}; };
let Some(new_len) = length_add(out_len, incr) else { let Some(new_len) = length_add(out_len, incr) else {
#[cold] #[cold]
fn stop(single_count: usize, double_count: usize, preferred_quote: Quote) -> EscapeLayout { fn stop(
EscapeLayout { quote: choose_quote(single_count, double_count, preferred_quote).0, len: None } single_count: usize,
double_count: usize,
preferred_quote: Quote,
) -> EscapeLayout {
EscapeLayout {
quote: choose_quote(single_count, double_count, preferred_quote).0,
len: None,
}
} }
return stop(single_count, double_count, preferred_quote); return stop(single_count, double_count, preferred_quote);
}; };

21
rustpython_original/literal/src/float.rs
View File

@ -7,7 +7,7 @@ pub fn parse_str(literal: &str) -> Option<f64> {
} }
pub fn parse_bytes(literal: &[u8]) -> Option<f64> { pub fn parse_bytes(literal: &[u8]) -> Option<f64> {
parse_inner(trim_slice(literal, |b| b.is_ascii_whitespace())) parse_inner(trim_slice(literal, u8::is_ascii_whitespace))
} }
fn trim_slice<T>(v: &[T], mut trim: impl FnMut(&T) -> bool) -> &[T] { fn trim_slice<T>(v: &[T], mut trim: impl FnMut(&T) -> bool) -> &[T] {
@ -72,7 +72,7 @@ pub fn format_fixed(precision: usize, magnitude: f64, case: Case, alternate_form
} }
magnitude if magnitude.is_nan() => format_nan(case), magnitude if magnitude.is_nan() => format_nan(case),
magnitude if magnitude.is_infinite() => format_inf(case), magnitude if magnitude.is_infinite() => format_inf(case),
_ => "".to_string(), _ => String::new(),
} }
} }
@ -99,7 +99,7 @@ pub fn format_exponent(
} }
magnitude if magnitude.is_nan() => format_nan(case), magnitude if magnitude.is_nan() => format_nan(case),
magnitude if magnitude.is_infinite() => format_inf(case), magnitude if magnitude.is_infinite() => format_inf(case),
_ => "".to_string(), _ => String::new(),
} }
} }
@ -132,6 +132,11 @@ fn remove_trailing_decimal_point(s: String) -> String {
s s
} }
#[allow(
clippy::cast_sign_loss,
clippy::cast_possible_truncation,
clippy::cast_possible_wrap
)]
pub fn format_general( pub fn format_general(
precision: usize, precision: usize,
magnitude: f64, magnitude: f64,
@ -145,7 +150,7 @@ pub fn format_general(
let mut parts = r_exp.splitn(2, 'e'); let mut parts = r_exp.splitn(2, 'e');
let base = parts.next().unwrap(); let base = parts.next().unwrap();
let exponent = parts.next().unwrap().parse::<i64>().unwrap(); let exponent = parts.next().unwrap().parse::<i64>().unwrap();
if exponent < -4 || exponent + (always_shows_fract as i64) >= (precision as i64) { if exponent < -4 || exponent + i64::from(always_shows_fract) >= (precision as i64) {
let e = match case { let e = match case {
Case::Lower => 'e', Case::Lower => 'e',
Case::Upper => 'E', Case::Upper => 'E',
@ -164,7 +169,7 @@ pub fn format_general(
} }
magnitude if magnitude.is_nan() => format_nan(case), magnitude if magnitude.is_nan() => format_nan(case),
magnitude if magnitude.is_infinite() => format_inf(case), magnitude if magnitude.is_infinite() => format_inf(case),
_ => "".to_string(), _ => String::new(),
} }
} }
@ -231,7 +236,7 @@ pub fn from_hex(s: &str) -> Option<f64> {
if !has_p && has_dot { if !has_p && has_dot {
hex.push_str("p0"); hex.push_str("p0");
} else if !has_p && !has_dot { } else if !has_p && !has_dot {
hex.push_str(".p0") hex.push_str(".p0");
} }
hexf_parse::parse_hexf64(hex.as_str(), false).ok() hexf_parse::parse_hexf64(hex.as_str(), false).ok()
@ -261,6 +266,7 @@ pub fn to_hex(value: f64) -> String {
} }
#[test] #[test]
#[allow(clippy::float_cmp)]
fn test_to_hex() { fn test_to_hex() {
use rand::Rng; use rand::Rng;
for _ in 0..20000 { for _ in 0..20000 {
@ -273,7 +279,8 @@ fn test_to_hex() {
// println!("{} -> {}", f, hex); // println!("{} -> {}", f, hex);
let roundtrip = hexf_parse::parse_hexf64(&hex, false).unwrap(); let roundtrip = hexf_parse::parse_hexf64(&hex, false).unwrap();
// println!(" -> {}", roundtrip); // println!(" -> {}", roundtrip);
assert!(f == roundtrip, "{} {} {}", f, hex, roundtrip);
assert_eq!(f, roundtrip, "{f} {hex} {roundtrip}");
} }
} }

140
rustpython_original/literal/src/format.rs
View File

@ -121,7 +121,7 @@ impl FormatParse for FormatGrouping {
} }
} }
#[derive(Debug, PartialEq)] #[derive(Copy, Clone, Debug, PartialEq)]
pub enum FormatType { pub enum FormatType {
String, String,
Binary, Binary,
@ -311,8 +311,13 @@ impl FormatSpec {
.collect::<String>() .collect::<String>()
} }
#[allow(
clippy::cast_possible_wrap,
clippy::cast_possible_truncation,
clippy::cast_sign_loss
)]
fn add_magnitude_separators_for_char( fn add_magnitude_separators_for_char(
magnitude_str: String, magnitude_str: &str,
inter: i32, inter: i32,
sep: char, sep: char,
disp_digit_cnt: i32, disp_digit_cnt: i32,
@ -324,11 +329,16 @@ impl FormatSpec {
let int_digit_cnt = disp_digit_cnt - dec_digit_cnt; let int_digit_cnt = disp_digit_cnt - dec_digit_cnt;
let mut result = FormatSpec::separate_integer(magnitude_int_str, inter, sep, int_digit_cnt); let mut result = FormatSpec::separate_integer(magnitude_int_str, inter, sep, int_digit_cnt);
if let Some(part) = parts.next() { if let Some(part) = parts.next() {
result.push_str(&format!(".{part}")) result.push_str(&format!(".{part}"));
} }
result result
} }
#[allow(
clippy::cast_sign_loss,
clippy::cast_possible_wrap,
clippy::cast_possible_truncation
)]
fn separate_integer( fn separate_integer(
magnitude_str: String, magnitude_str: String,
inter: i32, inter: i32,
@ -336,7 +346,7 @@ impl FormatSpec {
disp_digit_cnt: i32, disp_digit_cnt: i32,
) -> String { ) -> String {
let magnitude_len = magnitude_str.len() as i32; let magnitude_len = magnitude_str.len() as i32;
let offset = (disp_digit_cnt % (inter + 1) == 0) as i32; let offset = i32::from(disp_digit_cnt % (inter + 1) == 0);
let disp_digit_cnt = disp_digit_cnt + offset; let disp_digit_cnt = disp_digit_cnt + offset;
let pad_cnt = disp_digit_cnt - magnitude_len; let pad_cnt = disp_digit_cnt - magnitude_len;
let sep_cnt = disp_digit_cnt / (inter + 1); let sep_cnt = disp_digit_cnt / (inter + 1);
@ -353,9 +363,14 @@ impl FormatSpec {
} }
} }
#[allow(
clippy::cast_sign_loss,
clippy::cast_possible_truncation,
clippy::cast_possible_wrap
)]
fn insert_separator(mut magnitude_str: String, inter: i32, sep: char, sep_cnt: i32) -> String { fn insert_separator(mut magnitude_str: String, inter: i32, sep: char, sep_cnt: i32) -> String {
let magnitude_len = magnitude_str.len() as i32; let magnitude_len = magnitude_str.len() as i32;
for i in 1..sep_cnt + 1 { for i in 1..=sep_cnt {
magnitude_str.insert((magnitude_len - inter * i) as usize, sep); magnitude_str.insert((magnitude_len - inter * i) as usize, sep);
} }
magnitude_str magnitude_str
@ -396,6 +411,7 @@ impl FormatSpec {
} }
} }
#[allow(clippy::cast_possible_wrap, clippy::cast_possible_truncation)]
fn add_magnitude_separators(&self, magnitude_str: String, prefix: &str) -> String { fn add_magnitude_separators(&self, magnitude_str: String, prefix: &str) -> String {
match &self.grouping_option { match &self.grouping_option {
Some(fg) => { Some(fg) => {
@ -408,7 +424,7 @@ impl FormatSpec {
let width = self.width.unwrap_or(magnitude_len) as i32 - prefix.len() as i32; let width = self.width.unwrap_or(magnitude_len) as i32 - prefix.len() as i32;
let disp_digit_cnt = cmp::max(width, magnitude_len as i32); let disp_digit_cnt = cmp::max(width, magnitude_len as i32);
FormatSpec::add_magnitude_separators_for_char( FormatSpec::add_magnitude_separators_for_char(
magnitude_str, &magnitude_str,
inter, inter,
sep, sep,
disp_digit_cnt, disp_digit_cnt,
@ -431,7 +447,7 @@ impl FormatSpec {
| FormatType::Character, | FormatType::Character,
) => self.format_int(&BigInt::from_u8(x).unwrap()), ) => self.format_int(&BigInt::from_u8(x).unwrap()),
Some(FormatType::Exponent(_) | FormatType::FixedPoint(_) | FormatType::Percentage) => { Some(FormatType::Exponent(_) | FormatType::FixedPoint(_) | FormatType::Percentage) => {
self.format_float(x as f64) self.format_float(f64::from(x))
} }
None => { None => {
let first_letter = (input.to_string().as_bytes()[0] as char).to_uppercase(); let first_letter = (input.to_string().as_bytes()[0] as char).to_uppercase();
@ -452,17 +468,19 @@ impl FormatSpec {
*case, *case,
self.alternate_form, self.alternate_form,
)), )),
Some(FormatType::Decimal) Some(
| Some(FormatType::Binary) FormatType::Decimal
| Some(FormatType::Octal) | FormatType::Binary
| Some(FormatType::Hex(_)) | FormatType::Octal
| Some(FormatType::String) | FormatType::Hex(_)
| Some(FormatType::Character) | FormatType::String
| Some(FormatType::Number(Case::Upper)) => { | FormatType::Character
| FormatType::Number(Case::Upper),
) => {
let ch = char::from(self.format_type.as_ref().unwrap()); let ch = char::from(self.format_type.as_ref().unwrap());
Err(FormatSpecError::UnknownFormatCode(ch, "float")) Err(FormatSpecError::UnknownFormatCode(ch, "float"))
} }
Some(FormatType::GeneralFormat(case)) | Some(FormatType::Number(case)) => { Some(FormatType::GeneralFormat(case) | FormatType::Number(case)) => {
let precision = if precision == 0 { 1 } else { precision }; let precision = if precision == 0 { 1 } else { precision };
Ok(float::format_general( Ok(float::format_general(
precision, precision,
@ -513,11 +531,17 @@ impl FormatSpec {
} }
}; };
let magnitude_str = self.add_magnitude_separators(raw_magnitude_str?, sign_str); let magnitude_str = self.add_magnitude_separators(raw_magnitude_str?, sign_str);
self.format_sign_and_align(&AsciiStr::new(&magnitude_str), sign_str, FormatAlign::Right) Ok(
self.format_sign_and_align(
&AsciiStr::new(&magnitude_str),
sign_str,
FormatAlign::Right,
),
)
} }
#[inline] #[inline]
fn format_int_radix(&self, magnitude: BigInt, radix: u32) -> Result<String, FormatSpecError> { fn format_int_radix(&self, magnitude: &BigInt, radix: u32) -> Result<String, FormatSpecError> {
match self.precision { match self.precision {
Some(_) => Err(FormatSpecError::PrecisionNotAllowed), Some(_) => Err(FormatSpecError::PrecisionNotAllowed),
None => Ok(magnitude.to_str_radix(radix)), None => Ok(magnitude.to_str_radix(radix)),
@ -539,19 +563,21 @@ impl FormatSpec {
"" ""
}; };
let raw_magnitude_str = match self.format_type { let raw_magnitude_str = match self.format_type {
Some(FormatType::Binary) => self.format_int_radix(magnitude, 2), Some(FormatType::Binary) => self.format_int_radix(&magnitude, 2),
Some(FormatType::Decimal) => self.format_int_radix(magnitude, 10), Some(FormatType::Decimal) => self.format_int_radix(&magnitude, 10),
Some(FormatType::Octal) => self.format_int_radix(magnitude, 8), Some(FormatType::Octal) => self.format_int_radix(&magnitude, 8),
Some(FormatType::Hex(Case::Lower)) => self.format_int_radix(magnitude, 16), Some(FormatType::Hex(Case::Lower)) => self.format_int_radix(&magnitude, 16),
Some(FormatType::Hex(Case::Upper)) => match self.precision { Some(FormatType::Hex(Case::Upper)) => {
Some(_) => Err(FormatSpecError::PrecisionNotAllowed), if self.precision.is_some() {
None => { Err(FormatSpecError::PrecisionNotAllowed)
} else {
let mut result = magnitude.to_str_radix(16); let mut result = magnitude.to_str_radix(16);
result.make_ascii_uppercase(); result.make_ascii_uppercase();
Ok(result) Ok(result)
} }
}, }
Some(FormatType::Number(Case::Lower)) => self.format_int_radix(magnitude, 10),
Some(FormatType::Number(Case::Lower)) => self.format_int_radix(&magnitude, 10),
Some(FormatType::Number(Case::Upper)) => { Some(FormatType::Number(Case::Upper)) => {
Err(FormatSpecError::UnknownFormatCode('N', "int")) Err(FormatSpecError::UnknownFormatCode('N', "int"))
} }
@ -560,18 +586,20 @@ impl FormatSpec {
(Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")), (Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")),
(_, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")), (_, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")),
(_, _) => match num.to_u32() { (_, _) => match num.to_u32() {
Some(n) if n <= 0x10ffff => Ok(std::char::from_u32(n).unwrap().to_string()), Some(n) if n <= 0x0010_ffff => Ok(std::char::from_u32(n).unwrap().to_string()),
Some(_) | None => Err(FormatSpecError::CodeNotInRange), Some(_) | None => Err(FormatSpecError::CodeNotInRange),
}, },
}, },
Some(FormatType::GeneralFormat(_)) Some(
| Some(FormatType::FixedPoint(_)) FormatType::GeneralFormat(_)
| Some(FormatType::Exponent(_)) | FormatType::FixedPoint(_)
| Some(FormatType::Percentage) => match num.to_f64() { | FormatType::Exponent(_)
| FormatType::Percentage,
) => match num.to_f64() {
Some(float) => return self.format_float(float), Some(float) => return self.format_float(float),
_ => Err(FormatSpecError::UnableToConvert), _ => Err(FormatSpecError::UnableToConvert),
}, },
None => self.format_int_radix(magnitude, 10), None => self.format_int_radix(&magnitude, 10),
}?; }?;
let format_sign = self.sign.unwrap_or(FormatSign::Minus); let format_sign = self.sign.unwrap_or(FormatSign::Minus);
let sign_str = match num.sign() { let sign_str = match num.sign() {
@ -584,11 +612,11 @@ impl FormatSpec {
}; };
let sign_prefix = format!("{sign_str}{prefix}"); let sign_prefix = format!("{sign_str}{prefix}");
let magnitude_str = self.add_magnitude_separators(raw_magnitude_str, &sign_prefix); let magnitude_str = self.add_magnitude_separators(raw_magnitude_str, &sign_prefix);
self.format_sign_and_align( Ok(self.format_sign_and_align(
&AsciiStr::new(&magnitude_str), &AsciiStr::new(&magnitude_str),
&sign_prefix, &sign_prefix,
FormatAlign::Right, FormatAlign::Right,
) ))
} }
pub fn format_string<T>(&self, s: &T) -> Result<String, FormatSpecError> pub fn format_string<T>(&self, s: &T) -> Result<String, FormatSpecError>
@ -597,14 +625,13 @@ impl FormatSpec {
{ {
self.validate_format(FormatType::String)?; self.validate_format(FormatType::String)?;
match self.format_type { match self.format_type {
Some(FormatType::String) | None => self Some(FormatType::String) | None => {
.format_sign_and_align(s, "", FormatAlign::Left) let mut value = self.format_sign_and_align(s, "", FormatAlign::Left);
.map(|mut value| { if let Some(precision) = self.precision {
if let Some(precision) = self.precision { value.truncate(precision);
value.truncate(precision); }
} Ok(value)
value }
}),
_ => { _ => {
let ch = char::from(self.format_type.as_ref().unwrap()); let ch = char::from(self.format_type.as_ref().unwrap());
Err(FormatSpecError::UnknownFormatCode(ch, "str")) Err(FormatSpecError::UnknownFormatCode(ch, "str"))
@ -612,12 +639,13 @@ impl FormatSpec {
} }
} }
#[allow(clippy::cast_possible_wrap, clippy::cast_possible_truncation)]
fn format_sign_and_align<T>( fn format_sign_and_align<T>(
&self, &self,
magnitude_str: &T, magnitude_str: &T,
sign_str: &str, sign_str: &str,
default_align: FormatAlign, default_align: FormatAlign,
) -> Result<String, FormatSpecError> ) -> String
where where
T: CharLen + Deref<Target = str>, T: CharLen + Deref<Target = str>,
{ {
@ -629,8 +657,8 @@ impl FormatSpec {
cmp::max(0, (w as i32) - (num_chars as i32) - (sign_str.len() as i32)) cmp::max(0, (w as i32) - (num_chars as i32) - (sign_str.len() as i32))
}); });
let magnitude_str = magnitude_str.deref(); let magnitude_str = &**magnitude_str;
Ok(match align { match align {
FormatAlign::Left => format!( FormatAlign::Left => format!(
"{}{}{}", "{}{}{}",
sign_str, sign_str,
@ -658,7 +686,7 @@ impl FormatSpec {
FormatSpec::compute_fill_string(fill_char, right_fill_chars_needed); FormatSpec::compute_fill_string(fill_char, right_fill_chars_needed);
format!("{left_fill_string}{sign_str}{magnitude_str}{right_fill_string}") format!("{left_fill_string}{sign_str}{magnitude_str}{right_fill_string}")
} }
}) }
} }
} }
@ -801,7 +829,7 @@ impl FieldName {
let mut parts = Vec::new(); let mut parts = Vec::new();
while let Some(part) = FieldNamePart::parse_part(&mut chars)? { while let Some(part) = FieldNamePart::parse_part(&mut chars)? {
parts.push(part) parts.push(part);
} }
Ok(FieldName { field_type, parts }) Ok(FieldName { field_type, parts })
@ -851,10 +879,10 @@ impl FormatString {
cur_text = remaining; cur_text = remaining;
} }
Err(err) => { Err(err) => {
return if !result_string.is_empty() { return if result_string.is_empty() {
Ok((FormatPart::Literal(result_string), cur_text))
} else {
Err(err) Err(err)
} else {
Ok((FormatPart::Literal(result_string), cur_text))
}; };
} }
} }
@ -910,20 +938,18 @@ impl FormatString {
} else if c == '{' { } else if c == '{' {
if nested { if nested {
return Err(FormatParseError::InvalidFormatSpecifier); return Err(FormatParseError::InvalidFormatSpecifier);
} else {
nested = true;
left.push(c);
continue;
} }
nested = true;
left.push(c);
continue;
} else if c == '}' { } else if c == '}' {
if nested { if nested {
nested = false; nested = false;
left.push(c); left.push(c);
continue; continue;
} else {
end_bracket_pos = Some(idx);
break;
} }
end_bracket_pos = Some(idx);
break;
} else { } else {
left.push(c); left.push(c);
} }

11
rustpython_original/ruff_python_ast/src/nodes.rs
View File

@ -901,6 +901,7 @@ impl From<ExprFormattedValue> for Expr {
/// Transforms a value prior to formatting it. /// Transforms a value prior to formatting it.
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, is_macro::Is)] #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, is_macro::Is)]
#[repr(i8)] #[repr(i8)]
#[allow(clippy::cast_possible_wrap)]
pub enum ConversionFlag { pub enum ConversionFlag {
/// No conversion /// No conversion
None = -1, // CPython uses -1 None = -1, // CPython uses -1
@ -1138,7 +1139,7 @@ impl BoolOp {
pub const fn and(&self) -> Option<BoolOpAnd> { pub const fn and(&self) -> Option<BoolOpAnd> {
match self { match self {
BoolOp::And => Some(BoolOpAnd), BoolOp::And => Some(BoolOpAnd),
_ => None, BoolOp::Or => None,
} }
} }
@ -1146,7 +1147,7 @@ impl BoolOp {
pub const fn or(&self) -> Option<BoolOpOr> { pub const fn or(&self) -> Option<BoolOpOr> {
match self { match self {
BoolOp::Or => Some(BoolOpOr), BoolOp::Or => Some(BoolOpOr),
_ => None, BoolOp::And => None,
} }
} }
} }
@ -2107,7 +2108,7 @@ pub struct Decorator {
/// `defaults` and `kw_defaults` fields are removed and the default values are placed under each `arg_with_default` typed argument. /// `defaults` and `kw_defaults` fields are removed and the default values are placed under each `arg_with_default` typed argument.
/// `vararg` and `kwarg` are still typed as `arg` because they never can have a default value. /// `vararg` and `kwarg` are still typed as `arg` because they never can have a default value.
/// ///
/// The matching Python style AST type is [PythonArguments]. While [PythonArguments] has ordered `kwonlyargs` fields by /// The matching Python style AST type is [`PythonArguments`]. While [`PythonArguments`] has ordered `kwonlyargs` fields by
/// default existence, [Arguments] has location-ordered kwonlyargs fields. /// default existence, [Arguments] has location-ordered kwonlyargs fields.
/// ///
/// NOTE: This type is different from original Python AST. /// NOTE: This type is different from original Python AST.
@ -2200,14 +2201,14 @@ impl Arguments {
self.posonlyargs self.posonlyargs
.iter() .iter()
.chain(self.args.iter()) .chain(self.args.iter())
.filter_map(|arg| arg.default.as_ref().map(|e| e.as_ref())) .filter_map(|arg| arg.default.as_ref().map(std::convert::AsRef::as_ref))
} }
#[allow(clippy::type_complexity)] #[allow(clippy::type_complexity)]
pub fn split_kwonlyargs(&self) -> (Vec<&Arg>, Vec<(&Arg, &Expr)>) { pub fn split_kwonlyargs(&self) -> (Vec<&Arg>, Vec<(&Arg, &Expr)>) {
let mut args = Vec::new(); let mut args = Vec::new();
let mut with_defaults = Vec::new(); let mut with_defaults = Vec::new();
for arg in self.kwonlyargs.iter() { for arg in &self.kwonlyargs {
if let Some(ref default) = arg.default { if let Some(ref default) = arg.default {
with_defaults.push((arg.as_arg(), &**default)); with_defaults.push((arg.as_arg(), &**default));
} else { } else {

6
rustpython_original/ruff_python_parser/src/context.rs
View File

@ -2,7 +2,7 @@ use ruff_python_ast::{self as ast, Expr, ExprContext};
pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr { pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
match expr { match expr {
Expr::Name(ast::ExprName { id, range, .. }) => ast::ExprName { id, ctx, range }.into(), Expr::Name(ast::ExprName { id, range, .. }) => ast::ExprName { range, id, ctx }.into(),
Expr::Tuple(ast::ExprTuple { elts, range, .. }) => ast::ExprTuple { Expr::Tuple(ast::ExprTuple { elts, range, .. }) => ast::ExprTuple {
elts: elts.into_iter().map(|elt| set_context(elt, ctx)).collect(), elts: elts.into_iter().map(|elt| set_context(elt, ctx)).collect(),
range, range,
@ -19,9 +19,9 @@ pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
Expr::Attribute(ast::ExprAttribute { Expr::Attribute(ast::ExprAttribute {
value, attr, range, .. value, attr, range, ..
}) => ast::ExprAttribute { }) => ast::ExprAttribute {
range,
value, value,
attr, attr,
range,
ctx, ctx,
} }
.into(), .into(),
@ -31,9 +31,9 @@ pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
range, range,
.. ..
}) => ast::ExprSubscript { }) => ast::ExprSubscript {
range,
value, value,
slice, slice,
range,
ctx, ctx,
} }
.into(), .into(),

76
rustpython_original/ruff_python_parser/src/function.rs
View File

@ -1,3 +1,4 @@
use std::hash::BuildHasherDefault;
// Contains functions that perform validation and parsing of arguments and parameters. // Contains functions that perform validation and parsing of arguments and parameters.
// Checks apply both to functions and to lambdas. // Checks apply both to functions and to lambdas.
use crate::lexer::{LexicalError, LexicalErrorType}; use crate::lexer::{LexicalError, LexicalErrorType};
@ -15,10 +16,10 @@ pub(crate) fn validate_arguments(arguments: &ast::Arguments) -> Result<(), Lexic
let mut all_arg_names = FxHashSet::with_capacity_and_hasher( let mut all_arg_names = FxHashSet::with_capacity_and_hasher(
arguments.posonlyargs.len() arguments.posonlyargs.len()
+ arguments.args.len() + arguments.args.len()
+ arguments.vararg.is_some() as usize + usize::from(arguments.vararg.is_some())
+ arguments.kwonlyargs.len() + arguments.kwonlyargs.len()
+ arguments.kwarg.is_some() as usize, + usize::from(arguments.kwarg.is_some()),
Default::default(), BuildHasherDefault::default(),
); );
let posonlyargs = arguments.posonlyargs.iter(); let posonlyargs = arguments.posonlyargs.iter();
@ -79,49 +80,46 @@ pub(crate) fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentLis
let mut keywords = vec![]; let mut keywords = vec![];
let mut keyword_names = let mut keyword_names =
FxHashSet::with_capacity_and_hasher(func_args.len(), Default::default()); FxHashSet::with_capacity_and_hasher(func_args.len(), BuildHasherDefault::default());
let mut double_starred = false; let mut double_starred = false;
for (name, value) in func_args { for (name, value) in func_args {
match name { if let Some((start, end, name)) = name {
Some((start, end, name)) => { // Check for duplicate keyword arguments in the call.
// Check for duplicate keyword arguments in the call. if let Some(keyword_name) = &name {
if let Some(keyword_name) = &name { if !keyword_names.insert(keyword_name.to_string()) {
if !keyword_names.insert(keyword_name.to_string()) {
return Err(LexicalError {
error: LexicalErrorType::DuplicateKeywordArgumentError(
keyword_name.to_string(),
),
location: start,
});
}
} else {
double_starred = true;
}
keywords.push(ast::Keyword {
arg: name,
value,
range: TextRange::new(start, end),
});
}
None => {
// Positional arguments mustn't follow keyword arguments.
if !keywords.is_empty() && !is_starred(&value) {
return Err(LexicalError { return Err(LexicalError {
error: LexicalErrorType::PositionalArgumentError, error: LexicalErrorType::DuplicateKeywordArgumentError(
location: value.start(), keyword_name.to_string(),
),
location: start,
}); });
}
} else {
double_starred = true;
}
keywords.push(ast::Keyword {
arg: name,
value,
range: TextRange::new(start, end),
});
} else {
// Positional arguments mustn't follow keyword arguments.
if !keywords.is_empty() && !is_starred(&value) {
return Err(LexicalError {
error: LexicalErrorType::PositionalArgumentError,
location: value.start(),
});
// Allow starred arguments after keyword arguments but // Allow starred arguments after keyword arguments but
// not after double-starred arguments. // not after double-starred arguments.
} else if double_starred { } else if double_starred {
return Err(LexicalError { return Err(LexicalError {
error: LexicalErrorType::UnpackedArgumentError, error: LexicalErrorType::UnpackedArgumentError,
location: value.start(), location: value.start(),
}); });
}
args.push(value);
} }
args.push(value);
} }
} }
Ok(ArgumentList { args, keywords }) Ok(ArgumentList { args, keywords })

77
rustpython_original/ruff_python_parser/src/lexer.rs
View File

@ -143,6 +143,11 @@ impl<'source> Lexer<'source> {
/// Create a new lexer from T and a starting location. You probably want to use /// Create a new lexer from T and a starting location. You probably want to use
/// [`lex`] instead. /// [`lex`] instead.
pub fn new(input: &'source str, mode: Mode) -> Self { pub fn new(input: &'source str, mode: Mode) -> Self {
assert!(
u32::try_from(input.len()).is_ok(),
"Lexer only supports files with a size up to 4GB"
);
let mut lxr = Lexer { let mut lxr = Lexer {
state: State::AfterNewline, state: State::AfterNewline,
nesting: 0, nesting: 0,
@ -351,7 +356,7 @@ impl<'source> Lexer<'source> {
/// Consume a sequence of numbers with the given radix, /// Consume a sequence of numbers with the given radix,
/// the digits can be decorated with underscores /// the digits can be decorated with underscores
/// like this: '1_2_3_4' == '1234' /// like this: '`1_2_3_4`' == '1234'
fn radix_run(&mut self, first: Option<char>, radix: Radix) -> Cow<'source, str> { fn radix_run(&mut self, first: Option<char>, radix: Radix) -> Cow<'source, str> {
let start = if let Some(first) = first { let start = if let Some(first) = first {
self.offset() - first.text_len() self.offset() - first.text_len()
@ -384,13 +389,13 @@ impl<'source> Lexer<'source> {
} }
/// Lex a single comment. /// Lex a single comment.
fn lex_comment(&mut self) -> Result<Tok, LexicalError> { fn lex_comment(&mut self) -> Tok {
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
debug_assert_eq!(self.cursor.previous(), '#'); debug_assert_eq!(self.cursor.previous(), '#');
self.cursor.eat_while(|c| !matches!(c, '\n' | '\r')); self.cursor.eat_while(|c| !matches!(c, '\n' | '\r'));
return Ok(Tok::Comment(self.token_text().to_string())); Tok::Comment(self.token_text().to_string())
} }
/// Lex a single magic command. /// Lex a single magic command.
@ -418,10 +423,10 @@ impl<'source> Lexer<'source> {
self.cursor.bump(); self.cursor.bump();
self.cursor.bump(); self.cursor.bump();
continue; continue;
} else {
self.cursor.bump();
value.push('\\');
} }
self.cursor.bump();
value.push('\\');
} }
'\n' | '\r' | EOF_CHAR => { '\n' | '\r' | EOF_CHAR => {
return Tok::MagicCommand { kind, value }; return Tok::MagicCommand { kind, value };
@ -507,7 +512,7 @@ impl<'source> Lexer<'source> {
pub fn next_token(&mut self) -> LexResult { pub fn next_token(&mut self) -> LexResult {
// Return dedent tokens until the current indentation level matches the indentation of the next token. // Return dedent tokens until the current indentation level matches the indentation of the next token.
if let Some(indentation) = self.pending_indentation.take() { if let Some(indentation) = self.pending_indentation.take() {
if let Ok(Ordering::Greater) = self.indentations.current().try_compare(&indentation) { if let Ok(Ordering::Greater) = self.indentations.current().try_compare(indentation) {
self.pending_indentation = Some(indentation); self.pending_indentation = Some(indentation);
self.indentations.pop(); self.indentations.pop();
return Ok((Tok::Dedent, TextRange::empty(self.offset()))); return Ok((Tok::Dedent, TextRange::empty(self.offset())));
@ -601,7 +606,7 @@ impl<'source> Lexer<'source> {
&mut self, &mut self,
indentation: Indentation, indentation: Indentation,
) -> Result<Option<Spanned>, LexicalError> { ) -> Result<Option<Spanned>, LexicalError> {
let token = match self.indentations.current().try_compare(&indentation) { let token = match self.indentations.current().try_compare(indentation) {
// Dedent // Dedent
Ok(Ordering::Greater) => { Ok(Ordering::Greater) => {
self.indentations.pop(); self.indentations.pop();
@ -656,7 +661,7 @@ impl<'source> Lexer<'source> {
let token = match c { let token = match c {
c if is_ascii_identifier_start(c) => self.lex_identifier(c)?, c if is_ascii_identifier_start(c) => self.lex_identifier(c)?,
'0'..='9' => self.lex_number(c)?, '0'..='9' => self.lex_number(c)?,
'#' => return self.lex_comment().map(|token| (token, self.token_range())), '#' => return Ok((self.lex_comment(), self.token_range())),
'"' | '\'' => self.lex_string(StringKind::String, c)?, '"' | '\'' => self.lex_string(StringKind::String, c)?,
'=' => { '=' => {
if self.cursor.eat_char('=') { if self.cursor.eat_char('=') {
@ -900,6 +905,8 @@ impl<'source> Lexer<'source> {
&self.source[self.token_range()] &self.source[self.token_range()]
} }
// Lexer doesn't allow files larger than 4GB
#[allow(clippy::cast_possible_truncation)]
#[inline] #[inline]
fn offset(&self) -> TextSize { fn offset(&self) -> TextSize {
TextSize::new(self.source.len() as u32) - self.cursor.text_len() TextSize::new(self.source.len() as u32) - self.cursor.text_len()
@ -1153,7 +1160,7 @@ mod tests {
} }
fn assert_jupyter_magic_line_continuation_with_eol(eol: &str) { fn assert_jupyter_magic_line_continuation_with_eol(eol: &str) {
let source = format!("%matplotlib \\{} --inline", eol); let source = format!("%matplotlib \\{eol} --inline");
let tokens = lex_jupyter_source(&source); let tokens = lex_jupyter_source(&source);
assert_eq!( assert_eq!(
tokens, tokens,
@ -1164,7 +1171,7 @@ mod tests {
}, },
Tok::Newline Tok::Newline
] ]
) );
} }
#[test] #[test]
@ -1183,7 +1190,7 @@ mod tests {
} }
fn assert_jupyter_magic_line_continuation_with_eol_and_eof(eol: &str) { fn assert_jupyter_magic_line_continuation_with_eol_and_eof(eol: &str) {
let source = format!("%matplotlib \\{}", eol); let source = format!("%matplotlib \\{eol}");
let tokens = lex_jupyter_source(&source); let tokens = lex_jupyter_source(&source);
assert_eq!( assert_eq!(
tokens, tokens,
@ -1194,7 +1201,7 @@ mod tests {
}, },
Tok::Newline Tok::Newline
] ]
) );
} }
#[test] #[test]
@ -1220,52 +1227,52 @@ mod tests {
tokens, tokens,
vec![ vec![
Tok::MagicCommand { Tok::MagicCommand {
value: "".to_string(), value: String::new(),
kind: MagicKind::Magic, kind: MagicKind::Magic,
}, },
Tok::Newline, Tok::Newline,
Tok::MagicCommand { Tok::MagicCommand {
value: "".to_string(), value: String::new(),
kind: MagicKind::Magic2, kind: MagicKind::Magic2,
}, },
Tok::Newline, Tok::Newline,
Tok::MagicCommand { Tok::MagicCommand {
value: "".to_string(), value: String::new(),
kind: MagicKind::Shell, kind: MagicKind::Shell,
}, },
Tok::Newline, Tok::Newline,
Tok::MagicCommand { Tok::MagicCommand {
value: "".to_string(), value: String::new(),
kind: MagicKind::ShCap, kind: MagicKind::ShCap,
}, },
Tok::Newline, Tok::Newline,
Tok::MagicCommand { Tok::MagicCommand {
value: "".to_string(), value: String::new(),
kind: MagicKind::Help, kind: MagicKind::Help,
}, },
Tok::Newline, Tok::Newline,
Tok::MagicCommand { Tok::MagicCommand {
value: "".to_string(), value: String::new(),
kind: MagicKind::Help2, kind: MagicKind::Help2,
}, },
Tok::Newline, Tok::Newline,
Tok::MagicCommand { Tok::MagicCommand {
value: "".to_string(), value: String::new(),
kind: MagicKind::Paren, kind: MagicKind::Paren,
}, },
Tok::Newline, Tok::Newline,
Tok::MagicCommand { Tok::MagicCommand {
value: "".to_string(), value: String::new(),
kind: MagicKind::Quote, kind: MagicKind::Quote,
}, },
Tok::Newline, Tok::Newline,
Tok::MagicCommand { Tok::MagicCommand {
value: "".to_string(), value: String::new(),
kind: MagicKind::Quote2, kind: MagicKind::Quote2,
}, },
Tok::Newline, Tok::Newline,
] ]
) );
} }
#[test] #[test]
@ -1346,7 +1353,7 @@ mod tests {
}, },
Tok::Newline, Tok::Newline,
] ]
) );
} }
#[test] #[test]
fn test_jupyter_magic_indentation() { fn test_jupyter_magic_indentation() {
@ -1371,7 +1378,7 @@ if True:
Tok::Newline, Tok::Newline,
Tok::Dedent, Tok::Dedent,
] ]
) );
} }
#[test] #[test]
@ -1424,13 +1431,13 @@ baz = %matplotlib \
}, },
Tok::Newline, Tok::Newline,
] ]
) );
} }
fn assert_no_jupyter_magic(tokens: &[Tok]) { fn assert_no_jupyter_magic(tokens: &[Tok]) {
for tok in tokens { for tok in tokens {
if let Tok::MagicCommand { .. } = tok { if let Tok::MagicCommand { .. } = tok {
panic!("Unexpected magic command token: {:?}", tok) panic!("Unexpected magic command token: {tok:?}")
} }
} }
} }
@ -1475,7 +1482,7 @@ def f(arg=%timeit a = b):
value: BigInt::from(123), value: BigInt::from(123),
}, },
Tok::Int { Tok::Int {
value: BigInt::from(1234567890), value: BigInt::from(1_234_567_890),
}, },
Tok::Float { value: 0.2 }, Tok::Float { value: 0.2 },
Tok::Float { value: 100.0 }, Tok::Float { value: 100.0 },
@ -1851,13 +1858,13 @@ def f(arg=%timeit a = b):
} }
fn assert_string_continuation_with_eol(eol: &str) { fn assert_string_continuation_with_eol(eol: &str) {
let source = format!("\"abc\\{}def\"", eol); let source = format!("\"abc\\{eol}def\"");
let tokens = lex_source(&source); let tokens = lex_source(&source);
assert_eq!( assert_eq!(
tokens, tokens,
vec![str_tok(&format!("abc\\{}def", eol)), Tok::Newline] vec![str_tok(&format!("abc\\{eol}def")), Tok::Newline]
) );
} }
#[test] #[test]
@ -1879,23 +1886,23 @@ def f(arg=%timeit a = b):
fn test_escape_unicode_name() { fn test_escape_unicode_name() {
let source = r#""\N{EN SPACE}""#; let source = r#""\N{EN SPACE}""#;
let tokens = lex_source(source); let tokens = lex_source(source);
assert_eq!(tokens, vec![str_tok(r"\N{EN SPACE}"), Tok::Newline]) assert_eq!(tokens, vec![str_tok(r"\N{EN SPACE}"), Tok::Newline]);
} }
fn assert_triple_quoted(eol: &str) { fn assert_triple_quoted(eol: &str) {
let source = format!("\"\"\"{0} test string{0} \"\"\"", eol); let source = format!("\"\"\"{eol} test string{eol} \"\"\"");
let tokens = lex_source(&source); let tokens = lex_source(&source);
assert_eq!( assert_eq!(
tokens, tokens,
vec![ vec![
Tok::String { Tok::String {
value: format!("{0} test string{0} ", eol), value: format!("{eol} test string{eol} "),
kind: StringKind::String, kind: StringKind::String,
triple_quoted: true, triple_quoted: true,
}, },
Tok::Newline, Tok::Newline,
] ]
) );
} }
#[test] #[test]

6
rustpython_original/ruff_python_parser/src/lexer/cursor.rs
View File

@ -28,13 +28,13 @@ impl<'a> Cursor<'a> {
} }
/// Peeks the next character from the input stream without consuming it. /// Peeks the next character from the input stream without consuming it.
/// Returns [EOF_CHAR] if the file is at the end of the file. /// Returns [`EOF_CHAR`] if the file is at the end of the file.
pub(super) fn first(&self) -> char { pub(super) fn first(&self) -> char {
self.chars.clone().next().unwrap_or(EOF_CHAR) self.chars.clone().next().unwrap_or(EOF_CHAR)
} }
/// Peeks the second character from the input stream without consuming it. /// Peeks the second character from the input stream without consuming it.
/// Returns [EOF_CHAR] if the position is past the end of the file. /// Returns [`EOF_CHAR`] if the position is past the end of the file.
pub(super) fn second(&self) -> char { pub(super) fn second(&self) -> char {
let mut chars = self.chars.clone(); let mut chars = self.chars.clone();
chars.next(); chars.next();
@ -57,7 +57,7 @@ impl<'a> Cursor<'a> {
} }
pub(super) fn start_token(&mut self) { pub(super) fn start_token(&mut self) {
self.source_length = self.text_len() self.source_length = self.text_len();
} }
pub(super) fn is_eof(&self) -> bool { pub(super) fn is_eof(&self) -> bool {

15
rustpython_original/ruff_python_parser/src/lexer/indentation.rs
View File

@ -44,7 +44,7 @@ impl Indentation {
#[cfg(test)] #[cfg(test)]
pub(super) const fn new(column: Column, character: Character) -> Self { pub(super) const fn new(column: Column, character: Character) -> Self {
Self { character, column } Self { column, character }
} }
#[must_use] #[must_use]
@ -67,10 +67,7 @@ impl Indentation {
} }
} }
pub(super) fn try_compare( pub(super) fn try_compare(self, other: Indentation) -> Result<Ordering, UnexpectedIndentation> {
&self,
other: &Indentation,
) -> Result<Ordering, UnexpectedIndentation> {
let column_ordering = self.column.cmp(&other.column); let column_ordering = self.column.cmp(&other.column);
let character_ordering = self.character.cmp(&other.character); let character_ordering = self.character.cmp(&other.character);
@ -94,7 +91,7 @@ pub(super) struct Indentations {
impl Indentations { impl Indentations {
pub(super) fn push(&mut self, indent: Indentation) { pub(super) fn push(&mut self, indent: Indentation) {
debug_assert_eq!(self.current().try_compare(&indent), Ok(Ordering::Less)); debug_assert_eq!(self.current().try_compare(indent), Ok(Ordering::Less));
self.stack.push(indent); self.stack.push(indent);
} }
@ -120,10 +117,10 @@ mod tests {
fn indentation_try_compare() { fn indentation_try_compare() {
let tab = Indentation::new(Column::new(8), Character::new(1)); let tab = Indentation::new(Column::new(8), Character::new(1));
assert_eq!(tab.try_compare(&tab), Ok(Ordering::Equal)); assert_eq!(tab.try_compare(tab), Ok(Ordering::Equal));
let two_tabs = Indentation::new(Column::new(16), Character::new(2)); let two_tabs = Indentation::new(Column::new(16), Character::new(2));
assert_eq!(two_tabs.try_compare(&tab), Ok(Ordering::Greater)); assert_eq!(two_tabs.try_compare(tab), Ok(Ordering::Greater));
assert_eq!(tab.try_compare(&two_tabs), Ok(Ordering::Less)); assert_eq!(tab.try_compare(two_tabs), Ok(Ordering::Less));
} }
} }

200
rustpython_original/ruff_python_parser/src/gen/parse.rs → rustpython_original/ruff_python_parser/src/parse.rs
View File

@ -1,4 +1,202 @@
// This file was originally generated from asdl by a python script, but we now edit it manually use crate::lexer::{lex, lex_starts_at, LexResult};
use crate::{parse_tokens, Mode, ParseError, ParseErrorType};
use ruff_python_ast as ast;
use ruff_python_ast::Ranged;
use ruff_text_size::TextSize;
/// Parse Python code string to implementor's type.
///
/// # Example
///
/// For example, parsing a simple function definition and a call to that function:
///
/// ```
/// use ruff_python_parser::{self as parser, Parse};
/// use ruff_python_ast as ast;
/// let source = r#"
/// def foo():
/// return 42
///
/// print(foo())
/// "#;
/// let program = ast::Suite::parse(source, "<embedded>");
/// assert!(program.is_ok());
/// ```
///
/// Parsing a single expression denoting the addition of two numbers, but this time specifying a different,
/// somewhat silly, location:
///
/// ```
/// # use ruff_text_size::TextSize;
/// # use ruff_python_ast as ast;
/// # use ruff_python_parser::{self as parser, Parse};
///
/// let expr = ast::Expr::parse_starts_at("1 + 2", "<embedded>", TextSize::from(400));
/// assert!(expr.is_ok());
pub trait Parse
where
Self: Sized,
{
const MODE: Mode;
fn parse(source: &str, source_path: &str) -> Result<Self, ParseError> {
let tokens = lex(source, Self::MODE);
Self::parse_tokens(tokens, source_path)
}
fn parse_without_path(source: &str) -> Result<Self, ParseError> {
Self::parse(source, "<unknown>")
}
fn parse_starts_at(
source: &str,
source_path: &str,
offset: TextSize,
) -> Result<Self, ParseError> {
let tokens = lex_starts_at(source, Self::MODE, offset);
Self::parse_tokens(tokens, source_path)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError>;
}
impl Parse for ast::ModModule {
const MODE: Mode = Mode::Module;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
match parse_tokens(lxr, Mode::Module, source_path)? {
ast::Mod::Module(m) => Ok(m),
_ => unreachable!("Mode::Module doesn't return other variant"),
}
}
}
impl Parse for ast::ModExpression {
const MODE: Mode = Mode::Expression;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
match parse_tokens(lxr, Mode::Expression, source_path)? {
ast::Mod::Expression(m) => Ok(m),
_ => unreachable!("Mode::Module doesn't return other variant"),
}
}
}
impl Parse for ast::ModInteractive {
const MODE: Mode = Mode::Interactive;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
match parse_tokens(lxr, Mode::Interactive, source_path)? {
ast::Mod::Interactive(m) => Ok(m),
_ => unreachable!("Mode::Module doesn't return other variant"),
}
}
}
impl Parse for ast::Suite {
const MODE: Mode = Mode::Module;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
Ok(ast::ModModule::parse_tokens(lxr, source_path)?.body)
}
}
impl Parse for ast::Stmt {
const MODE: Mode = Mode::Module;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
let mut statements = ast::ModModule::parse_tokens(lxr, source_path)?.body;
let statement = match statements.len() {
0 => {
return Err(ParseError {
error: ParseErrorType::Eof,
offset: TextSize::default(),
source_path: source_path.to_owned(),
})
}
1 => statements.pop().unwrap(),
_ => {
return Err(ParseError {
error: ParseErrorType::InvalidToken,
offset: statements[1].range().start(),
source_path: source_path.to_owned(),
})
}
};
Ok(statement)
}
}
impl Parse for ast::Expr {
const MODE: Mode = Mode::Expression;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
Ok(*ast::ModExpression::parse_tokens(lxr, source_path)?.body)
}
}
impl Parse for ast::Identifier {
const MODE: Mode = Mode::Expression;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
let expr = ast::Expr::parse_tokens(lxr, source_path)?;
match expr {
ast::Expr::Name(name) => {
let range = name.range();
Ok(ast::Identifier::new(name.id, range))
}
expr => Err(ParseError {
error: ParseErrorType::InvalidToken,
offset: expr.range().start(),
source_path: source_path.to_owned(),
}),
}
}
}
impl Parse for ast::Constant {
const MODE: Mode = Mode::Expression;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
let expr = ast::Expr::parse_tokens(lxr, source_path)?;
match expr {
ast::Expr::Constant(c) => Ok(c.value),
expr => Err(ParseError {
error: ParseErrorType::InvalidToken,
offset: expr.range().start(),
source_path: source_path.to_owned(),
}),
}
}
}
impl Parse for ast::StmtFunctionDef { impl Parse for ast::StmtFunctionDef {
const MODE: Mode = Mode::Module; const MODE: Mode = Mode::Module;

222
rustpython_original/ruff_python_parser/src/parser.rs
View File

@ -1,4 +1,4 @@
//! Contains the interface to the Python ruff_python_parser. //! Contains the interface to the Python `ruff_python_parser`.
//! //!
//! Functions in this module can be used to parse Python code into an [Abstract Syntax Tree] //! Functions in this module can be used to parse Python code into an [Abstract Syntax Tree]
//! (AST) that is then transformed into bytecode. //! (AST) that is then transformed into bytecode.
@ -16,210 +16,15 @@ use std::{fmt, iter};
use itertools::Itertools; use itertools::Itertools;
pub(super) use lalrpop_util::ParseError as LalrpopError; pub(super) use lalrpop_util::ParseError as LalrpopError;
use ruff_text_size::TextSize; use ruff_text_size::{TextRange, TextSize};
use crate::lexer::{lex, lex_starts_at};
use crate::{ use crate::{
lexer::{self, LexResult, LexicalError, LexicalErrorType}, lexer::{self, LexResult, LexicalError, LexicalErrorType},
python, python,
token::Tok, token::Tok,
Mode, Mode, Parse,
}; };
use ruff_python_ast::{self as ast, Ranged}; use ruff_python_ast as ast;
/// Parse Python code string to implementor's type.
///
/// # Example
///
/// For example, parsing a simple function definition and a call to that function:
///
/// ```
/// use ruff_python_parser::{self as parser, Parse};
/// use ruff_python_ast as ast;
/// let source = r#"
/// def foo():
/// return 42
///
/// print(foo())
/// "#;
/// let program = ast::Suite::parse(source, "<embedded>");
/// assert!(program.is_ok());
/// ```
///
/// Parsing a single expression denoting the addition of two numbers, but this time specifying a different,
/// somewhat silly, location:
///
/// ```
/// # use ruff_text_size::TextSize;
/// # use ruff_python_ast as ast;
/// # use ruff_python_parser::{self as parser, Parse};
///
/// let expr = ast::Expr::parse_starts_at("1 + 2", "<embedded>", TextSize::from(400));
/// assert!(expr.is_ok());
pub trait Parse
where
Self: Sized,
{
const MODE: Mode;
fn parse(source: &str, source_path: &str) -> Result<Self, ParseError> {
let tokens = lex(source, Self::MODE);
Self::parse_tokens(tokens, source_path)
}
fn parse_without_path(source: &str) -> Result<Self, ParseError> {
Self::parse(source, "<unknown>")
}
fn parse_starts_at(
source: &str,
source_path: &str,
offset: TextSize,
) -> Result<Self, ParseError> {
let tokens = lex_starts_at(source, Self::MODE, offset);
Self::parse_tokens(tokens, source_path)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError>;
}
impl Parse for ast::ModModule {
const MODE: Mode = Mode::Module;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
match parse_tokens(lxr, Mode::Module, source_path)? {
ast::Mod::Module(m) => Ok(m),
_ => unreachable!("Mode::Module doesn't return other variant"),
}
}
}
impl Parse for ast::ModExpression {
const MODE: Mode = Mode::Expression;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
match parse_tokens(lxr, Mode::Expression, source_path)? {
ast::Mod::Expression(m) => Ok(m),
_ => unreachable!("Mode::Module doesn't return other variant"),
}
}
}
impl Parse for ast::ModInteractive {
const MODE: Mode = Mode::Interactive;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
match parse_tokens(lxr, Mode::Interactive, source_path)? {
ast::Mod::Interactive(m) => Ok(m),
_ => unreachable!("Mode::Module doesn't return other variant"),
}
}
}
impl Parse for ast::Suite {
const MODE: Mode = Mode::Module;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
Ok(ast::ModModule::parse_tokens(lxr, source_path)?.body)
}
}
impl Parse for ast::Stmt {
const MODE: Mode = Mode::Module;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
let mut statements = ast::ModModule::parse_tokens(lxr, source_path)?.body;
let statement = match statements.len() {
0 => {
return Err(ParseError {
error: ParseErrorType::Eof,
offset: TextSize::default(),
source_path: source_path.to_owned(),
})
}
1 => statements.pop().unwrap(),
_ => {
return Err(ParseError {
error: ParseErrorType::InvalidToken,
offset: statements[1].range().start(),
source_path: source_path.to_owned(),
})
}
};
Ok(statement)
}
}
impl Parse for ast::Expr {
const MODE: Mode = Mode::Expression;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
Ok(*ast::ModExpression::parse_tokens(lxr, source_path)?.body)
}
}
impl Parse for ast::Identifier {
const MODE: Mode = Mode::Expression;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
let expr = ast::Expr::parse_tokens(lxr, source_path)?;
match expr {
ast::Expr::Name(name) => {
let range = name.range();
Ok(ast::Identifier::new(name.id, range))
}
expr => Err(ParseError {
error: ParseErrorType::InvalidToken,
offset: expr.range().start(),
source_path: source_path.to_owned(),
}),
}
}
}
impl Parse for ast::Constant {
const MODE: Mode = Mode::Expression;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
let expr = ast::Expr::parse_tokens(lxr, source_path)?;
match expr {
ast::Expr::Constant(c) => Ok(c.value),
expr => Err(ParseError {
error: ParseErrorType::InvalidToken,
offset: expr.range().start(),
source_path: source_path.to_owned(),
}),
}
}
}
/// Parse a full Python program usually consisting of multiple lines. /// Parse a full Python program usually consisting of multiple lines.
/// ///
@ -241,7 +46,7 @@ impl Parse for ast::Constant {
/// let program = parser::parse_program(source, "<embedded>"); /// let program = parser::parse_program(source, "<embedded>");
/// assert!(program.is_ok()); /// assert!(program.is_ok());
/// ``` /// ```
#[deprecated = "Use ruff_python_ast::Suite::parse from rustpython_parser::Parse trait."] #[deprecated = "Use ruff_python_ast::Suite::parse from ruff_python_parser::Parse trait."]
pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, ParseError> { pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, ParseError> {
parse(source, Mode::Module, source_path).map(|top| match top { parse(source, Mode::Module, source_path).map(|top| match top {
ast::Mod::Module(ast::ModModule { body, .. }) => body, ast::Mod::Module(ast::ModModule { body, .. }) => body,
@ -265,7 +70,7 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
/// assert!(expr.is_ok()); /// assert!(expr.is_ok());
/// ///
/// ``` /// ```
#[deprecated = "Use ruff_python_ast::Expr::parse from rustpython_parser::Parse trait."] #[deprecated = "Use ruff_python_ast::Expr::parse from ruff_python_parser::Parse trait."]
pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> { pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> {
ast::Expr::parse(source, path) ast::Expr::parse(source, path)
} }
@ -287,7 +92,7 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
/// let expr = parse_expression_starts_at("1 + 2", "<embedded>", TextSize::from(400)); /// let expr = parse_expression_starts_at("1 + 2", "<embedded>", TextSize::from(400));
/// assert!(expr.is_ok()); /// assert!(expr.is_ok());
/// ``` /// ```
#[deprecated = "Use ruff_python_ast::Expr::parse_starts_at from rustpython_parser::Parse trait."] #[deprecated = "Use ruff_python_ast::Expr::parse_starts_at from ruff_python_parser::Parse trait."]
pub fn parse_expression_starts_at( pub fn parse_expression_starts_at(
source: &str, source: &str,
path: &str, path: &str,
@ -346,7 +151,7 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
parse_starts_at(source, mode, source_path, TextSize::default()) parse_starts_at(source, mode, source_path, TextSize::default())
} }
/// Parse the given Python source code using the specified [`Mode`] and [`Location`]. /// Parse the given Python source code using the specified [`Mode`] and [`TextSize`].
/// ///
/// This function allows to specify the location of the the source code, other than /// This function allows to specify the location of the the source code, other than
/// that, it behaves exactly like [`parse`]. /// that, it behaves exactly like [`parse`].
@ -413,7 +218,7 @@ fn parse_filtered_tokens(
mode: Mode, mode: Mode,
source_path: &str, source_path: &str,
) -> Result<ast::Mod, ParseError> { ) -> Result<ast::Mod, ParseError> {
let marker_token = (Tok::start_marker(mode), Default::default()); let marker_token = (Tok::start_marker(mode), TextRange::default());
let lexer = iter::once(Ok(marker_token)).chain(lxr); let lexer = iter::once(Ok(marker_token)).chain(lxr);
python::TopParser::new() python::TopParser::new()
.parse( .parse(
@ -574,14 +379,11 @@ impl ParseErrorType {
pub fn is_tab_error(&self) -> bool { pub fn is_tab_error(&self) -> bool {
matches!( matches!(
self, self,
ParseErrorType::Lexical(LexicalErrorType::TabError) ParseErrorType::Lexical(LexicalErrorType::TabError | LexicalErrorType::TabsAfterSpaces)
| ParseErrorType::Lexical(LexicalErrorType::TabsAfterSpaces)
) )
} }
} }
include!("gen/parse.rs");
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::Parse; use crate::Parse;
@ -933,7 +735,7 @@ type X[T: int, *Ts, **P] = (T, Ts, P)
type X[T: (int, str), *Ts, **P] = (T, Ts, P) type X[T: (int, str), *Ts, **P] = (T, Ts, P)
# soft keyword as alias name # soft keyword as alias name
type type = int type type = int
type match = int type match = int
type case = int type case = int
@ -1047,7 +849,7 @@ if 10 .real:
y = 100[no] y = 100[no]
y = 100(no) y = 100(no)
"#; "#;
assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap()) assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
} }
#[test] #[test]

56
rustpython_original/ruff_python_parser/src/python.lalrpop
View File

@ -228,7 +228,7 @@ RaiseStatement: ast::Stmt = {
}, },
<location:@L> "raise" <t:Test<"all">> <c:("from" <Test<"all">>)?> <end_location:@R> => { <location:@L> "raise" <t:Test<"all">> <c:("from" <Test<"all">>)?> <end_location:@R> => {
ast::Stmt::Raise( ast::Stmt::Raise(
ast::StmtRaise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x)), range: (location..end_location).into() } ast::StmtRaise { exc: Some(Box::new(t)), cause: c.map(Box::new), range: (location..end_location).into() }
) )
}, },
}; };
@ -285,7 +285,7 @@ ImportAsAlias<I>: ast::Alias = {
DottedName: ast::Identifier = { DottedName: ast::Identifier = {
<location:@L> <n:name> <end_location:@R> => ast::Identifier::new(n, (location..end_location).into()), <location:@L> <n:name> <end_location:@R> => ast::Identifier::new(n, (location..end_location).into()),
<location:@L> <n:name> <n2: ("." Identifier)+> <end_location:@R> => { <location:@L> <n:name> <n2: ("." Identifier)+> <end_location:@R> => {
let mut r = n.to_string(); let mut r = n;
for x in n2 { for x in n2 {
r.push('.'); r.push('.');
r.push_str(x.1.as_str()); r.push_str(x.1.as_str());
@ -315,7 +315,7 @@ AssertStatement: ast::Stmt = {
ast::Stmt::Assert( ast::Stmt::Assert(
ast::StmtAssert { ast::StmtAssert {
test: Box::new(test), test: Box::new(test),
msg: msg.map(|e| Box::new(e)), msg: msg.map(Box::new),
range: (location..end_location).into() range: (location..end_location).into()
} }
) )
@ -346,10 +346,10 @@ LineMagicExpr: ast::Expr = {
if mode == Mode::Jupyter { if mode == Mode::Jupyter {
// This should never occur as the lexer won't allow it. // This should never occur as the lexer won't allow it.
if !matches!(m.0, MagicKind::Magic | MagicKind::Shell) { if !matches!(m.0, MagicKind::Magic | MagicKind::Shell) {
Err(LexicalError { return Err(LexicalError {
error: LexicalErrorType::OtherError("expr line magics are only allowed for % and !".to_string()), error: LexicalErrorType::OtherError("expr line magics are only allowed for % and !".to_string()),
location, location,
})? })?;
} }
Ok(ast::Expr::LineMagic( Ok(ast::Expr::LineMagic(
ast::ExprLineMagic { ast::ExprLineMagic {
@ -678,42 +678,42 @@ MatchMappingEntry: (ast::Expr, ast::Pattern) = {
MappingPattern: ast::Pattern = { MappingPattern: ast::Pattern = {
<location:@L> "{" "}" <end_location:@R> => { <location:@L> "{" "}" <end_location:@R> => {
return ast::PatternMatchMapping { ast::PatternMatchMapping {
keys: vec![], keys: vec![],
patterns: vec![], patterns: vec![],
rest: None, rest: None,
range: (location..end_location).into() range: (location..end_location).into()
}.into(); }.into()
}, },
<location:@L> "{" <e:OneOrMore<MatchMappingEntry>> ","? "}" <end_location:@R> => { <location:@L> "{" <e:OneOrMore<MatchMappingEntry>> ","? "}" <end_location:@R> => {
let (keys, patterns) = e let (keys, patterns) = e
.into_iter() .into_iter()
.unzip(); .unzip();
return ast::PatternMatchMapping { ast::PatternMatchMapping {
keys, keys,
patterns, patterns,
rest: None, rest: None,
range: (location..end_location).into() range: (location..end_location).into()
}.into(); }.into()
}, },
<location:@L> "{" "**" <rest:Identifier> ","? "}" <end_location:@R> => { <location:@L> "{" "**" <rest:Identifier> ","? "}" <end_location:@R> => {
return ast::PatternMatchMapping { ast::PatternMatchMapping {
keys: vec![], keys: vec![],
patterns: vec![], patterns: vec![],
rest: Some(rest), rest: Some(rest),
range: (location..end_location).into() range: (location..end_location).into()
}.into(); }.into()
}, },
<location:@L> "{" <e:OneOrMore<MatchMappingEntry>> "," "**" <rest:Identifier> ","? "}" <end_location:@R> => { <location:@L> "{" <e:OneOrMore<MatchMappingEntry>> "," "**" <rest:Identifier> ","? "}" <end_location:@R> => {
let (keys, patterns) = e let (keys, patterns) = e
.into_iter() .into_iter()
.unzip(); .unzip();
return ast::PatternMatchMapping { ast::PatternMatchMapping {
keys, keys,
patterns, patterns,
rest: Some(rest), rest: Some(rest),
range: (location..end_location).into() range: (location..end_location).into()
}.into(); }.into()
}, },
} }
@ -822,8 +822,7 @@ IfStatement: ast::Stmt = {
let end_location = elif_else_clauses let end_location = elif_else_clauses
.last() .last()
.map(|last| last.end()) .map_or_else(|| body.last().unwrap().end(), Ranged::end);
.unwrap_or_else(|| body.last().unwrap().end());
ast::Stmt::If( ast::Stmt::If(
ast::StmtIf { test: Box::new(test), body, elif_else_clauses, range: (location..end_location).into() } ast::StmtIf { test: Box::new(test), body, elif_else_clauses, range: (location..end_location).into() }
@ -875,9 +874,9 @@ TryStatement: ast::Stmt = {
let finalbody = finalbody.unwrap_or_default(); let finalbody = finalbody.unwrap_or_default();
let end_location = finalbody let end_location = finalbody
.last() .last()
.map(|last| last.end()) .map(Ranged::end)
.or_else(|| orelse.last().map(|last| last.end())) .or_else(|| orelse.last().map(Ranged::end))
.or_else(|| handlers.last().map(|last| last.end())) .or_else(|| handlers.last().map(Ranged::end))
.unwrap(); .unwrap();
ast::Stmt::Try( ast::Stmt::Try(
ast::StmtTry { ast::StmtTry {
@ -895,8 +894,8 @@ TryStatement: ast::Stmt = {
let end_location = finalbody let end_location = finalbody
.last() .last()
.or_else(|| orelse.last()) .or_else(|| orelse.last())
.map(|last| last.end()) .map(Ranged::end)
.or_else(|| handlers.last().map(|last| last.end())) .or_else(|| handlers.last().map(Ranged::end))
.unwrap(); .unwrap();
ast::Stmt::TryStar( ast::Stmt::TryStar(
ast::StmtTryStar { ast::StmtTryStar {
@ -1016,7 +1015,7 @@ WithItem<Goal>: ast::WithItem = {
FuncDef: ast::Stmt = { FuncDef: ast::Stmt = {
<location:@L> <decorator_list:Decorator*> <is_async:"async"?> "def" <name:Identifier> <type_params:TypeParamList?> <args:Parameters> <r:("->" <Test<"all">>)?> ":" <body:Suite> => { <location:@L> <decorator_list:Decorator*> <is_async:"async"?> "def" <name:Identifier> <type_params:TypeParamList?> <args:Parameters> <r:("->" <Test<"all">>)?> ":" <body:Suite> => {
let args = Box::new(args); let args = Box::new(args);
let returns = r.map(|x| Box::new(x)); let returns = r.map(Box::new);
let end_location = body.last().unwrap().end(); let end_location = body.last().unwrap().end();
let type_comment = None; let type_comment = None;
if is_async.is_some() { if is_async.is_some() {
@ -1052,11 +1051,10 @@ Parameters: ast::Arguments = {
let range = (location..end_location).into(); let range = (location..end_location).into();
let args = a let args = a
.map(|mut arguments| { .map_or_else(|| ast::Arguments::empty(range), |mut arguments| {
arguments.range = range; arguments.range = range;
arguments arguments
}) });
.unwrap_or_else(|| ast::Arguments::empty(range));
Ok(args) Ok(args)
} }
@ -1180,10 +1178,10 @@ DoubleStarTypedParameter: ast::Arg = {
ParameterListStarArgs<ArgType, StarArgType, DoubleStarArgType>: (Option<Box<ast::Arg>>, Vec<ast::ArgWithDefault>, Option<Box<ast::Arg>>) = { ParameterListStarArgs<ArgType, StarArgType, DoubleStarArgType>: (Option<Box<ast::Arg>>, Vec<ast::ArgWithDefault>, Option<Box<ast::Arg>>) = {
<location:@L> "*" <va:StarArgType?> <kwonlyargs:("," <ParameterDef<ArgType>>)*> <kwarg:("," <KwargParameter<DoubleStarArgType>>)?> =>? { <location:@L> "*" <va:StarArgType?> <kwonlyargs:("," <ParameterDef<ArgType>>)*> <kwarg:("," <KwargParameter<DoubleStarArgType>>)?> =>? {
if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() { if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() {
Err(LexicalError { return Err(LexicalError {
error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()),
location, location,
})? })?;
} }
let kwarg = kwarg.flatten(); let kwarg = kwarg.flatten();
@ -1526,10 +1524,10 @@ Atom<Goal>: ast::Expr = {
<location:@L> "(" <left:(<OneOrMore<Test<"all">>> ",")?> <mid:NamedOrStarExpr> <right:("," <TestOrStarNamedExpr>)*> <trailing_comma:","?> ")" <end_location:@R> =>? { <location:@L> "(" <left:(<OneOrMore<Test<"all">>> ",")?> <mid:NamedOrStarExpr> <right:("," <TestOrStarNamedExpr>)*> <trailing_comma:","?> ")" <end_location:@R> =>? {
if left.is_none() && right.is_empty() && trailing_comma.is_none() { if left.is_none() && right.is_empty() && trailing_comma.is_none() {
if mid.is_starred_expr() { if mid.is_starred_expr() {
Err(LexicalError{ return Err(LexicalError{
error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
location: mid.start(), location: mid.start(),
})? })?;
} }
Ok(mid) Ok(mid)
} else { } else {
@ -1720,7 +1718,7 @@ OneOrMore<T>: Vec<T> = {
} }
}; };
/// Two or more items that are separted by `Sep` /// Two or more items that are separated by `Sep`
TwoOrMore<T, Sep>: Vec<T> = { TwoOrMore<T, Sep>: Vec<T> = {
<e1:T> Sep <e2:T> => vec![e1, e2], <e1:T> Sep <e2:T> => vec![e1, e2],
<mut v: TwoOrMore<T, Sep>> Sep <e:T> => { <mut v: TwoOrMore<T, Sep>> Sep <e:T> => {

64
rustpython_original/ruff_python_parser/src/python.rs
View File

@ -1,5 +1,5 @@
// auto-generated: "lalrpop 0.20.0" // auto-generated: "lalrpop 0.20.0"
// sha3: 9c49dc85355275f274dcc32e163c443c0dc567214c9725547e2218e9acd22577 // sha3: bf0ea34f78939474a89bc0d4b6e7c14f370a2d2cd2ca8b98bd5aefdae0e1d5f1
use num_bigint::BigInt; use num_bigint::BigInt;
use ruff_text_size::TextSize; use ruff_text_size::TextSize;
use ruff_python_ast::{self as ast, Ranged, MagicKind}; use ruff_python_ast::{self as ast, Ranged, MagicKind};
@ -31555,7 +31555,7 @@ fn __action59<
{ {
{ {
ast::Stmt::Raise( ast::Stmt::Raise(
ast::StmtRaise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x)), range: (location..end_location).into() } ast::StmtRaise { exc: Some(Box::new(t)), cause: c.map(Box::new), range: (location..end_location).into() }
) )
} }
} }
@ -31723,7 +31723,7 @@ fn __action70<
) -> ast::Identifier ) -> ast::Identifier
{ {
{ {
let mut r = n.to_string(); let mut r = n;
for x in n2 { for x in n2 {
r.push('.'); r.push('.');
r.push_str(x.1.as_str()); r.push_str(x.1.as_str());
@ -31784,7 +31784,7 @@ fn __action73<
ast::Stmt::Assert( ast::Stmt::Assert(
ast::StmtAssert { ast::StmtAssert {
test: Box::new(test), test: Box::new(test),
msg: msg.map(|e| Box::new(e)), msg: msg.map(Box::new),
range: (location..end_location).into() range: (location..end_location).into()
} }
) )
@ -31833,10 +31833,10 @@ fn __action75<
if mode == Mode::Jupyter { if mode == Mode::Jupyter {
// This should never occur as the lexer won't allow it. // This should never occur as the lexer won't allow it.
if !matches!(m.0, MagicKind::Magic | MagicKind::Shell) { if !matches!(m.0, MagicKind::Magic | MagicKind::Shell) {
Err(LexicalError { return Err(LexicalError {
error: LexicalErrorType::OtherError("expr line magics are only allowed for % and !".to_string()), error: LexicalErrorType::OtherError("expr line magics are only allowed for % and !".to_string()),
location, location,
})? })?;
} }
Ok(ast::Expr::LineMagic( Ok(ast::Expr::LineMagic(
ast::ExprLineMagic { ast::ExprLineMagic {
@ -32791,12 +32791,12 @@ fn __action133<
) -> ast::Pattern ) -> ast::Pattern
{ {
{ {
return ast::PatternMatchMapping { ast::PatternMatchMapping {
keys: vec![], keys: vec![],
patterns: vec![], patterns: vec![],
rest: None, rest: None,
range: (location..end_location).into() range: (location..end_location).into()
}.into(); }.into()
} }
} }
@ -32817,12 +32817,12 @@ fn __action134<
let (keys, patterns) = e let (keys, patterns) = e
.into_iter() .into_iter()
.unzip(); .unzip();
return ast::PatternMatchMapping { ast::PatternMatchMapping {
keys, keys,
patterns, patterns,
rest: None, rest: None,
range: (location..end_location).into() range: (location..end_location).into()
}.into(); }.into()
} }
} }
@ -32841,12 +32841,12 @@ fn __action135<
) -> ast::Pattern ) -> ast::Pattern
{ {
{ {
return ast::PatternMatchMapping { ast::PatternMatchMapping {
keys: vec![], keys: vec![],
patterns: vec![], patterns: vec![],
rest: Some(rest), rest: Some(rest),
range: (location..end_location).into() range: (location..end_location).into()
}.into(); }.into()
} }
} }
@ -32870,12 +32870,12 @@ fn __action136<
let (keys, patterns) = e let (keys, patterns) = e
.into_iter() .into_iter()
.unzip(); .unzip();
return ast::PatternMatchMapping { ast::PatternMatchMapping {
keys, keys,
patterns, patterns,
rest: Some(rest), rest: Some(rest),
range: (location..end_location).into() range: (location..end_location).into()
}.into(); }.into()
} }
} }
@ -33131,8 +33131,7 @@ fn __action146<
let end_location = elif_else_clauses let end_location = elif_else_clauses
.last() .last()
.map(|last| last.end()) .map_or_else(|| body.last().unwrap().end(), Ranged::end);
.unwrap_or_else(|| body.last().unwrap().end());
ast::Stmt::If( ast::Stmt::If(
ast::StmtIf { test: Box::new(test), body, elif_else_clauses, range: (location..end_location).into() } ast::StmtIf { test: Box::new(test), body, elif_else_clauses, range: (location..end_location).into() }
@ -33225,9 +33224,9 @@ fn __action149<
let finalbody = finalbody.unwrap_or_default(); let finalbody = finalbody.unwrap_or_default();
let end_location = finalbody let end_location = finalbody
.last() .last()
.map(|last| last.end()) .map(Ranged::end)
.or_else(|| orelse.last().map(|last| last.end())) .or_else(|| orelse.last().map(Ranged::end))
.or_else(|| handlers.last().map(|last| last.end())) .or_else(|| handlers.last().map(Ranged::end))
.unwrap(); .unwrap();
ast::Stmt::Try( ast::Stmt::Try(
ast::StmtTry { ast::StmtTry {
@ -33262,8 +33261,8 @@ fn __action150<
let end_location = finalbody let end_location = finalbody
.last() .last()
.or_else(|| orelse.last()) .or_else(|| orelse.last())
.map(|last| last.end()) .map(Ranged::end)
.or_else(|| handlers.last().map(|last| last.end())) .or_else(|| handlers.last().map(Ranged::end))
.unwrap(); .unwrap();
ast::Stmt::TryStar( ast::Stmt::TryStar(
ast::StmtTryStar { ast::StmtTryStar {
@ -33522,7 +33521,7 @@ fn __action162<
{ {
{ {
let args = Box::new(args); let args = Box::new(args);
let returns = r.map(|x| Box::new(x)); let returns = r.map(Box::new);
let end_location = body.last().unwrap().end(); let end_location = body.last().unwrap().end();
let type_comment = None; let type_comment = None;
if is_async.is_some() { if is_async.is_some() {
@ -33591,11 +33590,10 @@ fn __action165<
let range = (location..end_location).into(); let range = (location..end_location).into();
let args = a let args = a
.map(|mut arguments| { .map_or_else(|| ast::Arguments::empty(range), |mut arguments| {
arguments.range = range; arguments.range = range;
arguments arguments
}) });
.unwrap_or_else(|| ast::Arguments::empty(range));
Ok(args) Ok(args)
} }
@ -37001,10 +36999,10 @@ fn __action417<
{ {
{ {
if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() { if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() {
Err(LexicalError { return Err(LexicalError {
error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()),
location, location,
})? })?;
} }
let kwarg = kwarg.flatten(); let kwarg = kwarg.flatten();
@ -37118,10 +37116,10 @@ fn __action425<
{ {
{ {
if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() { if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() {
Err(LexicalError { return Err(LexicalError {
error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()),
location, location,
})? })?;
} }
let kwarg = kwarg.flatten(); let kwarg = kwarg.flatten();
@ -38469,10 +38467,10 @@ fn __action524<
{ {
if left.is_none() && right.is_empty() && trailing_comma.is_none() { if left.is_none() && right.is_empty() && trailing_comma.is_none() {
if mid.is_starred_expr() { if mid.is_starred_expr() {
Err(LexicalError{ return Err(LexicalError{
error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
location: mid.start(), location: mid.start(),
})? })?;
} }
Ok(mid) Ok(mid)
} else { } else {
@ -39137,10 +39135,10 @@ fn __action568<
{ {
if left.is_none() && right.is_empty() && trailing_comma.is_none() { if left.is_none() && right.is_empty() && trailing_comma.is_none() {
if mid.is_starred_expr() { if mid.is_starred_expr() {
Err(LexicalError{ return Err(LexicalError{
error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
location: mid.start(), location: mid.start(),
})? })?;
} }
Ok(mid) Ok(mid)
} else { } else {

14
rustpython_original/ruff_python_parser/src/soft_keywords.rs
View File

@ -15,7 +15,7 @@ use itertools::{Itertools, MultiPeek};
/// soft keyword tokens with `identifier` tokens if they are used as identifiers. /// soft keyword tokens with `identifier` tokens if they are used as identifiers.
/// ///
/// Handling soft keywords in this intermediary pass allows us to simplify both the lexer and /// Handling soft keywords in this intermediary pass allows us to simplify both the lexer and
/// ruff_python_parser, as neither of them need to be aware of soft keywords. /// `ruff_python_parser`, as neither of them need to be aware of soft keywords.
pub struct SoftKeywordTransformer<I> pub struct SoftKeywordTransformer<I>
where where
I: Iterator<Item = LexResult>, I: Iterator<Item = LexResult>,
@ -59,9 +59,7 @@ where
// (This is to avoid treating `match` or `case` as identifiers when annotated with // (This is to avoid treating `match` or `case` as identifiers when annotated with
// type hints.) type hints.) // type hints.) type hints.)
Tok::Match | Tok::Case => { Tok::Match | Tok::Case => {
if !self.start_of_line { if self.start_of_line {
next = Some(Ok((soft_to_name(tok), *range)));
} else {
let mut nesting = 0; let mut nesting = 0;
let mut first = true; let mut first = true;
let mut seen_colon = false; let mut seen_colon = false;
@ -86,6 +84,8 @@ where
if !seen_colon { if !seen_colon {
next = Some(Ok((soft_to_name(tok), *range))); next = Some(Ok((soft_to_name(tok), *range)));
} }
} else {
next = Some(Ok((soft_to_name(tok), *range)));
} }
} }
// For `type` all of the following conditions must be met: // For `type` all of the following conditions must be met:
@ -93,9 +93,7 @@ where
// 2. The type token is immediately followed by a name token. // 2. The type token is immediately followed by a name token.
// 3. The name token is eventually followed by an equality token. // 3. The name token is eventually followed by an equality token.
Tok::Type => { Tok::Type => {
if !self.start_of_line { if self.start_of_line {
next = Some(Ok((soft_to_name(tok), *range)));
} else {
let mut is_type_alias = false; let mut is_type_alias = false;
if let Some(Ok((tok, _))) = self.underlying.peek() { if let Some(Ok((tok, _))) = self.underlying.peek() {
if matches!( if matches!(
@ -126,6 +124,8 @@ where
if !is_type_alias { if !is_type_alias {
next = Some(Ok((soft_to_name(tok), *range))); next = Some(Ok((soft_to_name(tok), *range)));
} }
} else {
next = Some(Ok((soft_to_name(tok), *range)));
} }
} }
_ => (), // Not a soft keyword token _ => (), // Not a soft keyword token

224
rustpython_original/ruff_python_parser/src/string.rs
View File

@ -11,8 +11,9 @@ use ruff_text_size::{TextLen, TextRange, TextSize};
// we have to do the parsing here, manually. // we have to do the parsing here, manually.
use crate::{ use crate::{
lexer::{LexicalError, LexicalErrorType}, lexer::{LexicalError, LexicalErrorType},
parser::{LalrpopError, Parse, ParseError, ParseErrorType}, parser::{ParseError, ParseErrorType},
token::{StringKind, Tok}, token::{StringKind, Tok},
Parse,
}; };
// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798 // unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
@ -68,11 +69,6 @@ impl<'a> StringParser<'a> {
TextRange::new(start_location, self.location) TextRange::new(start_location, self.location)
} }
#[inline]
fn expr(&self, node: Expr) -> Expr {
node
}
fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> { fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
let mut p: u32 = 0u32; let mut p: u32 = 0u32;
let unicode_error = LexicalError::new(LexicalErrorType::UnicodeError, self.get_pos()); let unicode_error = LexicalError::new(LexicalErrorType::UnicodeError, self.get_pos());
@ -96,7 +92,7 @@ impl<'a> StringParser<'a> {
octet_content.push(first); octet_content.push(first);
while octet_content.len() < 3 { while octet_content.len() < 3 {
if let Some('0'..='7') = self.peek() { if let Some('0'..='7') = self.peek() {
octet_content.push(self.next_char().unwrap()) octet_content.push(self.next_char().unwrap());
} else { } else {
break; break;
} }
@ -157,7 +153,7 @@ impl<'a> StringParser<'a> {
'U' if !self.kind.is_any_bytes() => self.parse_unicode_literal(8)?, 'U' if !self.kind.is_any_bytes() => self.parse_unicode_literal(8)?,
'N' if !self.kind.is_any_bytes() => self.parse_unicode_name()?, 'N' if !self.kind.is_any_bytes() => self.parse_unicode_name()?,
// Special cases where the escape sequence is not a single character // Special cases where the escape sequence is not a single character
'\n' => return Ok("".to_string()), '\n' => return Ok(String::new()),
c => { c => {
if self.kind.is_any_bytes() && !c.is_ascii() { if self.kind.is_any_bytes() && !c.is_ascii() {
return Err(LexicalError { return Err(LexicalError {
@ -180,7 +176,10 @@ impl<'a> StringParser<'a> {
} }
fn parse_formatted_value(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> { fn parse_formatted_value(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
use FStringErrorType::*; use FStringErrorType::{
EmptyExpression, InvalidConversionFlag, InvalidExpression, MismatchedDelimiter,
UnclosedLbrace, Unmatched, UnterminatedString,
};
let mut expression = String::new(); let mut expression = String::new();
let mut spec = None; let mut spec = None;
@ -238,15 +237,10 @@ impl<'a> StringParser<'a> {
let start_location = self.get_pos(); let start_location = self.get_pos();
let parsed_spec = self.parse_spec(nested)?; let parsed_spec = self.parse_spec(nested)?;
spec = Some(Box::new( spec = Some(Box::new(Expr::from(ast::ExprJoinedStr {
self.expr( values: parsed_spec,
ast::ExprJoinedStr { range: self.range(start_location),
values: parsed_spec, })));
range: self.range(start_location),
}
.into(),
),
));
} }
'(' | '{' | '[' => { '(' | '{' | '[' => {
expression.push(ch); expression.push(ch);
@ -309,9 +303,21 @@ impl<'a> StringParser<'a> {
return Err(FStringError::new(EmptyExpression, self.get_pos()).into()); return Err(FStringError::new(EmptyExpression, self.get_pos()).into());
} }
let ret = if !self_documenting { let ret = if self_documenting {
vec![self.expr( // TODO: range is wrong but `self_documenting` needs revisiting beyond
ast::ExprFormattedValue { // ranges: https://github.com/astral-sh/ruff/issues/5970
vec![
Expr::from(ast::ExprConstant {
value: Constant::Str(expression.clone() + "="),
kind: None,
range: self.range(start_location),
}),
Expr::from(ast::ExprConstant {
value: trailing_seq.into(),
kind: None,
range: self.range(start_location),
}),
Expr::from(ast::ExprFormattedValue {
value: Box::new( value: Box::new(
parse_fstring_expr(&expression, start_location).map_err( parse_fstring_expr(&expression, start_location).map_err(
|e| { |e| {
@ -322,57 +328,30 @@ impl<'a> StringParser<'a> {
}, },
)?, )?,
), ),
conversion, conversion: if conversion == ConversionFlag::None && spec.is_none()
{
ConversionFlag::Repr
} else {
conversion
},
format_spec: spec, format_spec: spec,
range: self.range(start_location), range: self.range(start_location),
} }),
.into(),
)]
} else {
// TODO: range is wrong but `self_documenting` needs revisiting beyond
// ranges: https://github.com/astral-sh/ruff/issues/5970
vec![
self.expr(
ast::ExprConstant {
value: Constant::Str(expression.to_owned() + "="),
kind: None,
range: self.range(start_location),
}
.into(),
),
self.expr(
ast::ExprConstant {
value: trailing_seq.into(),
kind: None,
range: self.range(start_location),
}
.into(),
),
self.expr(
ast::ExprFormattedValue {
value: Box::new(
parse_fstring_expr(&expression, start_location).map_err(
|e| {
FStringError::new(
InvalidExpression(Box::new(e.error)),
start_location,
)
},
)?,
),
conversion: if conversion == ConversionFlag::None
&& spec.is_none()
{
ConversionFlag::Repr
} else {
conversion
},
format_spec: spec,
range: self.range(start_location),
}
.into(),
),
] ]
} else {
vec![Expr::from(ast::ExprFormattedValue {
value: Box::new(
parse_fstring_expr(&expression, start_location).map_err(|e| {
FStringError::new(
InvalidExpression(Box::new(e.error)),
start_location,
)
})?,
),
conversion,
format_spec: spec,
range: self.range(start_location),
})]
}; };
return Ok(ret); return Ok(ret);
} }
@ -380,7 +359,9 @@ impl<'a> StringParser<'a> {
expression.push(ch); expression.push(ch);
loop { loop {
let Some(c) = self.next_char() else { let Some(c) = self.next_char() else {
return Err(FStringError::new(UnterminatedString, self.get_pos()).into()); return Err(
FStringError::new(UnterminatedString, self.get_pos()).into()
);
}; };
expression.push(c); expression.push(c);
if c == ch { if c == ch {
@ -412,16 +393,11 @@ impl<'a> StringParser<'a> {
match next { match next {
'{' => { '{' => {
if !constant_piece.is_empty() { if !constant_piece.is_empty() {
spec_constructor.push( spec_constructor.push(Expr::from(ast::ExprConstant {
self.expr( value: constant_piece.drain(..).collect::<String>().into(),
ast::ExprConstant { kind: None,
value: constant_piece.drain(..).collect::<String>().into(), range: self.range(start_location),
kind: None, }));
range: self.range(start_location),
}
.into(),
),
);
} }
let parsed_expr = self.parse_fstring(nested + 1)?; let parsed_expr = self.parse_fstring(nested + 1)?;
spec_constructor.extend(parsed_expr); spec_constructor.extend(parsed_expr);
@ -438,22 +414,17 @@ impl<'a> StringParser<'a> {
self.next_char(); self.next_char();
} }
if !constant_piece.is_empty() { if !constant_piece.is_empty() {
spec_constructor.push( spec_constructor.push(Expr::from(ast::ExprConstant {
self.expr( value: constant_piece.drain(..).collect::<String>().into(),
ast::ExprConstant { kind: None,
value: constant_piece.drain(..).collect::<String>().into(), range: self.range(start_location),
kind: None, }));
range: self.range(start_location),
}
.into(),
),
);
} }
Ok(spec_constructor) Ok(spec_constructor)
} }
fn parse_fstring(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> { fn parse_fstring(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
use FStringErrorType::*; use FStringErrorType::{ExpressionNestedTooDeeply, SingleRbrace, UnclosedLbrace};
if nested >= 2 { if nested >= 2 {
return Err(FStringError::new(ExpressionNestedTooDeeply, self.get_pos()).into()); return Err(FStringError::new(ExpressionNestedTooDeeply, self.get_pos()).into());
@ -481,16 +452,11 @@ impl<'a> StringParser<'a> {
} }
} }
if !content.is_empty() { if !content.is_empty() {
values.push( values.push(Expr::from(ast::ExprConstant {
self.expr( value: content.drain(..).collect::<String>().into(),
ast::ExprConstant { kind: None,
value: content.drain(..).collect::<String>().into(), range: self.range(start_location),
kind: None, }));
range: self.range(start_location),
}
.into(),
),
);
} }
let parsed_values = self.parse_formatted_value(nested)?; let parsed_values = self.parse_formatted_value(nested)?;
@ -521,16 +487,11 @@ impl<'a> StringParser<'a> {
} }
if !content.is_empty() { if !content.is_empty() {
values.push( values.push(Expr::from(ast::ExprConstant {
self.expr( value: content.into(),
ast::ExprConstant { kind: None,
value: content.into(), range: self.range(start_location),
kind: None, }));
range: self.range(start_location),
}
.into(),
),
)
} }
Ok(values) Ok(values)
@ -558,14 +519,11 @@ impl<'a> StringParser<'a> {
} }
} }
Ok(self.expr( Ok(Expr::from(ast::ExprConstant {
ast::ExprConstant { value: Constant::Bytes(content.chars().map(|c| c as u8).collect()),
value: Constant::Bytes(content.chars().map(|c| c as u8).collect()), kind: None,
kind: None, range: self.range(start_location),
range: self.range(start_location), }))
}
.into(),
))
} }
fn parse_string(&mut self) -> Result<Expr, LexicalError> { fn parse_string(&mut self) -> Result<Expr, LexicalError> {
@ -579,14 +537,11 @@ impl<'a> StringParser<'a> {
ch => content.push(ch), ch => content.push(ch),
} }
} }
Ok(self.expr( Ok(Expr::from(ast::ExprConstant {
ast::ExprConstant { value: Constant::Str(content),
value: Constant::Str(content), kind: self.kind.is_unicode().then(|| "u".to_string()),
kind: self.kind.is_unicode().then(|| "u".to_string()), range: self.range(start_location),
range: self.range(start_location), }))
}
.into(),
))
} }
fn parse(&mut self) -> Result<Vec<Expr>, LexicalError> { fn parse(&mut self) -> Result<Vec<Expr>, LexicalError> {
@ -703,7 +658,7 @@ pub(crate) fn parse_strings(
if !current.is_empty() { if !current.is_empty() {
deduped.push(take_current(&mut current, current_start, current_end)); deduped.push(take_current(&mut current, current_start, current_end));
} }
deduped.push(value) deduped.push(value);
} }
Expr::Constant(ast::ExprConstant { Expr::Constant(ast::ExprConstant {
value: Constant::Str(inner), value: Constant::Str(inner),
@ -787,7 +742,11 @@ pub enum FStringErrorType {
impl std::fmt::Display for FStringErrorType { impl std::fmt::Display for FStringErrorType {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
use FStringErrorType::*; use FStringErrorType::{
EmptyExpression, ExpectedRbrace, ExpressionCannotInclude, ExpressionNestedTooDeeply,
InvalidConversionFlag, InvalidExpression, MismatchedDelimiter, SingleRbrace,
UnclosedLbrace, Unmatched, UnopenedRbrace, UnterminatedString,
};
match self { match self {
UnclosedLbrace => write!(f, "expecting '}}'"), UnclosedLbrace => write!(f, "expecting '}}'"),
UnopenedRbrace => write!(f, "Unopened '}}'"), UnopenedRbrace => write!(f, "Unopened '}}'"),
@ -820,7 +779,7 @@ impl std::fmt::Display for FStringErrorType {
} }
} }
impl From<FStringError> for LalrpopError<TextSize, Tok, LexicalError> { impl From<FStringError> for crate::parser::LalrpopError<TextSize, Tok, LexicalError> {
fn from(err: FStringError) -> Self { fn from(err: FStringError) -> Self {
lalrpop_util::ParseError::User { lalrpop_util::ParseError::User {
error: LexicalError { error: LexicalError {
@ -906,7 +865,10 @@ mod tests {
#[test] #[test]
fn test_parse_invalid_fstring() { fn test_parse_invalid_fstring() {
use FStringErrorType::*; use FStringErrorType::{
EmptyExpression, ExpressionNestedTooDeeply, InvalidConversionFlag, SingleRbrace,
UnclosedLbrace,
};
assert_eq!(parse_fstring_error("{5!a"), UnclosedLbrace); assert_eq!(parse_fstring_error("{5!a"), UnclosedLbrace);
assert_eq!(parse_fstring_error("{5!a1}"), UnclosedLbrace); assert_eq!(parse_fstring_error("{5!a1}"), UnclosedLbrace);
assert_eq!(parse_fstring_error("{5!"), UnclosedLbrace); assert_eq!(parse_fstring_error("{5!"), UnclosedLbrace);

7
rustpython_original/ruff_python_parser/src/token.rs
View File

@ -1,4 +1,4 @@
//! Token type for Python source code created by the lexer and consumed by the ruff_python_parser. //! Token type for Python source code created by the lexer and consumed by the `ruff_python_parser`.
//! //!
//! This module defines the tokens that the lexer recognizes. The tokens are //! This module defines the tokens that the lexer recognizes. The tokens are
//! loosely based on the token definitions found in the [CPython source]. //! loosely based on the token definitions found in the [CPython source].
@ -219,6 +219,7 @@ impl Tok {
impl fmt::Display for Tok { impl fmt::Display for Tok {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
#[allow(clippy::enum_glob_use)]
use Tok::*; use Tok::*;
match self { match self {
Name { name } => write!(f, "'{name}'"), Name { name } => write!(f, "'{name}'"),
@ -384,7 +385,7 @@ impl TryFrom<[char; 2]> for StringKind {
impl fmt::Display for StringKind { impl fmt::Display for StringKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
use StringKind::*; use StringKind::{Bytes, FString, RawBytes, RawFString, RawString, String, Unicode};
match self { match self {
String => f.write_str(""), String => f.write_str(""),
FString => f.write_str("f"), FString => f.write_str("f"),
@ -426,7 +427,7 @@ impl StringKind {
/// Returns the number of characters in the prefix. /// Returns the number of characters in the prefix.
pub fn prefix_len(&self) -> TextSize { pub fn prefix_len(&self) -> TextSize {
use StringKind::*; use StringKind::{Bytes, FString, RawBytes, RawFString, RawString, String, Unicode};
let len = match self { let len = match self {
String => 0, String => 0,
RawString | FString | Unicode | Bytes => 1, RawString | FString | Unicode | Bytes => 1,