Python
/
ruff
mirror of https://github.com/astral-sh/ruff
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix the ranges of constants inside f-strings (#33)

This commit is contained in:
David Szotten 2023-07-25 18:17:06 +01:00 committed by GitHub
parent 5ef4ccd632
commit 13196fc500
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 231 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap generated
View File

@ -12,7 +12,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..14, range: 2..13,
value: Str( value: Str(
"Hello world", "Hello world",
), ),

8
parser/src/snapshots/rustpython_parser__parser__tests__try.snap generated
View File

@ -80,7 +80,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 62..81, range: 64..71,
value: Str( value: Str(
"caught ", "caught ",
), ),
@ -89,7 +89,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 62..81, range: 71..80,
value: Call( value: Call(
ExprCall { ExprCall {
range: 72..79, range: 72..79,
@ -167,7 +167,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 114..133, range: 116..123,
value: Str( value: Str(
"caught ", "caught ",
), ),
@ -176,7 +176,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 114..133, range: 123..132,
value: Call( value: Call(
ExprCall { ExprCall {
range: 124..131, range: 124..131,

16
parser/src/snapshots/rustpython_parser__parser__tests__try_star.snap generated
View File

@ -184,7 +184,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 133..179, range: 135..142,
value: Str( value: Str(
"caught ", "caught ",
), ),
@ -193,7 +193,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 133..179, range: 142..151,
value: Call( value: Call(
ExprCall { ExprCall {
range: 143..150, range: 143..150,
@ -222,7 +222,7 @@ expression: parse_ast
), ),
Constant( Constant(
ExprConstant { ExprConstant {
range: 133..179, range: 151..164,
value: Str( value: Str(
" with nested ", " with nested ",
), ),
@ -231,7 +231,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 133..179, range: 164..178,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 165..177, range: 165..177,
@ -304,7 +304,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 213..259, range: 215..222,
value: Str( value: Str(
"caught ", "caught ",
), ),
@ -313,7 +313,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 213..259, range: 222..231,
value: Call( value: Call(
ExprCall { ExprCall {
range: 223..230, range: 223..230,
@ -342,7 +342,7 @@ expression: parse_ast
), ),
Constant( Constant(
ExprConstant { ExprConstant {
range: 213..259, range: 231..244,
value: Str( value: Str(
" with nested ", " with nested ",
), ),
@ -351,7 +351,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 213..259, range: 244..258,
value: Attribute( value: Attribute(
ExprAttribute { ExprAttribute {
range: 245..257, range: 245..257,

73
parser/src/snapshots/rustpython_parser__string__tests__fstring_constant_range.snap generated Normal file
View File

@ -0,0 +1,73 @@
---
source: parser/src/string.rs
expression: parse_ast
---
[
Expr(
StmtExpr {
range: 0..22,
value: JoinedStr(
ExprJoinedStr {
range: 0..22,
values: [
Constant(
ExprConstant {
range: 2..5,
value: Str(
"aaa",
),
kind: None,
},
),
FormattedValue(
ExprFormattedValue {
range: 5..10,
value: Name(
ExprName {
range: 6..9,
id: "bbb",
ctx: Load,
},
),
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 10..13,
value: Str(
"ccc",
),
kind: None,
},
),
FormattedValue(
ExprFormattedValue {
range: 13..18,
value: Name(
ExprName {
range: 14..17,
id: "ddd",
ctx: Load,
},
),
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 18..21,
value: Str(
"eee",
),
kind: None,
},
),
],
},
),
},
),
]

4
parser/src/snapshots/rustpython_parser__string__tests__fstring_escaped_character.snap generated
View File

@ -12,7 +12,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..8, range: 2..4,
value: Str( value: Str(
"\\", "\\",
), ),
@ -21,7 +21,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..8, range: 4..7,
value: Name( value: Name(
ExprName { ExprName {
range: 5..6, range: 5..6,

4
parser/src/snapshots/rustpython_parser__string__tests__fstring_escaped_newline.snap generated
View File

@ -12,7 +12,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..8, range: 2..4,
value: Str( value: Str(
"\n", "\n",
), ),
@ -21,7 +21,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..8, range: 4..7,
value: Name( value: Name(
ExprName { ExprName {
range: 5..6, range: 5..6,

4
parser/src/snapshots/rustpython_parser__string__tests__fstring_line_continuation.snap generated
View File

@ -12,7 +12,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..9, range: 3..5,
value: Str( value: Str(
"\\\n", "\\\n",
), ),
@ -21,7 +21,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..9, range: 5..8,
value: Name( value: Name(
ExprName { ExprName {
range: 6..7, range: 6..7,

6
parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_self_documenting_base.snap generated
View File

@ -5,7 +5,7 @@ expression: parse_ast
[ [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..10, range: 2..9,
value: Str( value: Str(
"user=", "user=",
), ),
@ -14,7 +14,7 @@ expression: parse_ast
), ),
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..10, range: 2..9,
value: Str( value: Str(
"", "",
), ),
@ -23,7 +23,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..10, range: 2..9,
value: Name( value: Name(
ExprName { ExprName {
range: 3..7, range: 3..7,

16
parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_self_documenting_base_more.snap generated
View File

@ -5,7 +5,7 @@ expression: parse_ast
[ [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..38, range: 2..6,
value: Str( value: Str(
"mix ", "mix ",
), ),
@ -14,7 +14,7 @@ expression: parse_ast
), ),
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..38, range: 6..13,
value: Str( value: Str(
"user=", "user=",
), ),
@ -23,7 +23,7 @@ expression: parse_ast
), ),
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..38, range: 6..13,
value: Str( value: Str(
"", "",
), ),
@ -32,7 +32,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..38, range: 6..13,
value: Name( value: Name(
ExprName { ExprName {
range: 7..11, range: 7..11,
@ -46,7 +46,7 @@ expression: parse_ast
), ),
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..38, range: 13..28,
value: Str( value: Str(
" with text and ", " with text and ",
), ),
@ -55,7 +55,7 @@ expression: parse_ast
), ),
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..38, range: 28..37,
value: Str( value: Str(
"second=", "second=",
), ),
@ -64,7 +64,7 @@ expression: parse_ast
), ),
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..38, range: 28..37,
value: Str( value: Str(
"", "",
), ),
@ -73,7 +73,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..38, range: 28..37,
value: Name( value: Name(
ExprName { ExprName {
range: 29..35, range: 29..35,

10
parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_self_documenting_format.snap generated
View File

@ -5,7 +5,7 @@ expression: parse_ast
[ [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..14, range: 2..13,
value: Str( value: Str(
"user=", "user=",
), ),
@ -14,7 +14,7 @@ expression: parse_ast
), ),
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..14, range: 2..13,
value: Str( value: Str(
"", "",
), ),
@ -23,7 +23,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..14, range: 2..13,
value: Name( value: Name(
ExprName { ExprName {
range: 3..7, range: 3..7,
@ -35,11 +35,11 @@ expression: parse_ast
format_spec: Some( format_spec: Some(
JoinedStr( JoinedStr(
ExprJoinedStr { ExprJoinedStr {
range: 0..14, range: 9..12,
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..14, range: 9..12,
value: Str( value: Str(
">10", ">10",
), ),

4
parser/src/snapshots/rustpython_parser__string__tests__fstring_unescaped_newline.snap generated
View File

@ -12,7 +12,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..11, range: 4..5,
value: Str( value: Str(
"\n", "\n",
), ),
@ -21,7 +21,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..11, range: 5..8,
value: Name( value: Name(
ExprName { ExprName {
range: 6..7, range: 6..7,

2
parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap generated
View File

@ -12,7 +12,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..17, range: 1..16,
value: Str( value: Str(
"Hello world", "Hello world",
), ),

2
parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap generated
View File

@ -12,7 +12,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..17, range: 1..16,
value: Str( value: Str(
"Hello world", "Hello world",
), ),

4
parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap generated
View File

@ -12,7 +12,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..22, range: 1..16,
value: Str( value: Str(
"Hello world", "Hello world",
), ),
@ -21,7 +21,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 9..22, range: 16..21,
value: Constant( value: Constant(
ExprConstant { ExprConstant {
range: 17..20, range: 17..20,

6
parser/src/snapshots/rustpython_parser__string__tests__parse_fstring.snap generated
View File

@ -5,7 +5,7 @@ expression: parse_ast
[ [
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..18, range: 2..5,
value: Name( value: Name(
ExprName { ExprName {
range: 3..4, range: 3..4,
@ -19,7 +19,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..18, range: 5..10,
value: Name( value: Name(
ExprName { ExprName {
range: 7..8, range: 7..8,
@ -33,7 +33,7 @@ expression: parse_ast
), ),
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..18, range: 10..17,
value: Str( value: Str(
"{foo}", "{foo}",
), ),

2
parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_equals.snap generated
View File

@ -5,7 +5,7 @@ expression: parse_ast
[ [
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..13, range: 2..12,
value: Compare( value: Compare(
ExprCompare { ExprCompare {
range: 3..11, range: 3..11,

6
parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_nested_spec.snap generated
View File

@ -5,7 +5,7 @@ expression: parse_ast
[ [
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..15, range: 2..14,
value: Name( value: Name(
ExprName { ExprName {
range: 3..6, range: 3..6,
@ -17,11 +17,11 @@ expression: parse_ast
format_spec: Some( format_spec: Some(
JoinedStr( JoinedStr(
ExprJoinedStr { ExprJoinedStr {
range: 0..15, range: 7..13,
values: [ values: [
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..15, range: 7..13,
value: Name( value: Name(
ExprName { ExprName {
range: 8..12, range: 8..12,

2
parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_equals.snap generated
View File

@ -5,7 +5,7 @@ expression: parse_ast
[ [
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..11, range: 2..10,
value: Compare( value: Compare(
ExprCompare { ExprCompare {
range: 3..9, range: 3..9,

6
parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_nested_spec.snap generated
View File

@ -5,7 +5,7 @@ expression: parse_ast
[ [
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..13, range: 2..12,
value: Name( value: Name(
ExprName { ExprName {
range: 3..6, range: 3..6,
@ -17,11 +17,11 @@ expression: parse_ast
format_spec: Some( format_spec: Some(
JoinedStr( JoinedStr(
ExprJoinedStr { ExprJoinedStr {
range: 0..13, range: 7..11,
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..13, range: 7..11,
value: Str( value: Str(
"spec", "spec",
), ),

6
parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_self_doc_prec_space.snap generated
View File

@ -5,7 +5,7 @@ expression: parse_ast
[ [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..10, range: 2..9,
value: Str( value: Str(
"x =", "x =",
), ),
@ -14,7 +14,7 @@ expression: parse_ast
), ),
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..10, range: 2..9,
value: Str( value: Str(
"", "",
), ),
@ -23,7 +23,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..10, range: 2..9,
value: Name( value: Name(
ExprName { ExprName {
range: 3..4, range: 3..4,

6
parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_self_doc_trailing_space.snap generated
View File

@ -5,7 +5,7 @@ expression: parse_ast
[ [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..10, range: 2..9,
value: Str( value: Str(
"x=", "x=",
), ),
@ -14,7 +14,7 @@ expression: parse_ast
), ),
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..10, range: 2..9,
value: Str( value: Str(
" ", " ",
), ),
@ -23,7 +23,7 @@ expression: parse_ast
), ),
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..10, range: 2..9,
value: Name( value: Name(
ExprName { ExprName {
range: 3..4, range: 3..4,

2
parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_yield_expr.snap generated
View File

@ -5,7 +5,7 @@ expression: parse_ast
[ [
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..10, range: 2..9,
value: Yield( value: Yield(
ExprYield { ExprYield {
range: 3..8, range: 3..8,

2
parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap generated
View File

@ -12,7 +12,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..18, range: 2..17,
value: Str( value: Str(
"Hello world", "Hello world",
), ),

2
parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap generated
View File

@ -12,7 +12,7 @@ expression: parse_ast
values: [ values: [
Constant( Constant(
ExprConstant { ExprConstant {
range: 0..22, range: 2..21,
value: Str( value: Str(
"Hello world!", "Hello world!",
), ),

2
parser/src/snapshots/rustpython_parser__string__tests__raw_fstring.snap generated
View File

@ -12,7 +12,7 @@ expression: parse_ast
values: [ values: [
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..7, range: 3..6,
value: Name( value: Name(
ExprName { ExprName {
range: 4..5, range: 4..5,

2
parser/src/snapshots/rustpython_parser__string__tests__triple_quoted_raw_fstring.snap generated
View File

@ -12,7 +12,7 @@ expression: parse_ast
values: [ values: [
FormattedValue( FormattedValue(
ExprFormattedValue { ExprFormattedValue {
range: 0..11, range: 5..8,
value: Name( value: Name(
ExprName { ExprName {
range: 6..7, range: 6..7,

169
parser/src/string.rs
View File

@ -11,6 +11,7 @@ use crate::{
token::{StringKind, Tok}, token::{StringKind, Tok},
}; };
use itertools::Itertools; use itertools::Itertools;
use rustpython_ast::Ranged;
use rustpython_parser_core::{ use rustpython_parser_core::{
text_size::{TextLen, TextSize}, text_size::{TextLen, TextSize},
ConversionFlag, ConversionFlag,
@ -20,21 +21,13 @@ use rustpython_parser_core::{
const MAX_UNICODE_NAME: usize = 88; const MAX_UNICODE_NAME: usize = 88;
struct StringParser<'a> { struct StringParser<'a> {
chars: std::iter::Peekable<std::str::Chars<'a>>, chars: std::str::Chars<'a>,
kind: StringKind, kind: StringKind,
start: TextSize,
end: TextSize,
location: TextSize, location: TextSize,
} }
impl<'a> StringParser<'a> { impl<'a> StringParser<'a> {
fn new( fn new(source: &'a str, kind: StringKind, triple_quoted: bool, start: TextSize) -> Self {
source: &'a str,
kind: StringKind,
triple_quoted: bool,
start: TextSize,
end: TextSize,
) -> Self {
let offset = kind.prefix_len() let offset = kind.prefix_len()
+ if triple_quoted { + if triple_quoted {
TextSize::from(3) TextSize::from(3)
@ -42,10 +35,8 @@ impl<'a> StringParser<'a> {
TextSize::from(1) TextSize::from(1)
}; };
Self { Self {
chars: source.chars().peekable(), chars: source.chars(),
kind, kind,
start,
end,
location: start + offset, location: start + offset,
} }
} }
@ -58,8 +49,15 @@ impl<'a> StringParser<'a> {
} }
#[inline] #[inline]
fn peek(&mut self) -> Option<&char> { fn peek(&mut self) -> Option<char> {
self.chars.peek() self.chars.clone().next()
}
#[inline]
fn peek2(&mut self) -> Option<char> {
let mut chars = self.chars.clone();
chars.next();
chars.next()
} }
#[inline] #[inline]
@ -68,12 +66,13 @@ impl<'a> StringParser<'a> {
} }
#[inline] #[inline]
fn expr(&self, node: Expr) -> Expr { fn range(&self, start_location: TextSize) -> TextRange {
node TextRange::new(start_location, self.location)
} }
fn range(&self) -> TextRange { #[inline]
TextRange::new(self.start, self.end) fn expr(&self, node: Expr) -> Expr {
node
} }
fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> { fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
@ -191,18 +190,20 @@ impl<'a> StringParser<'a> {
let mut conversion = ConversionFlag::None; let mut conversion = ConversionFlag::None;
let mut self_documenting = false; let mut self_documenting = false;
let mut trailing_seq = String::new(); let mut trailing_seq = String::new();
let location = self.get_pos(); let start_location = self.get_pos();
assert_eq!(self.next_char(), Some('{'));
while let Some(ch) = self.next_char() { while let Some(ch) = self.next_char() {
match ch { match ch {
// can be integrated better with the remaining code, but as a starting point ok // can be integrated better with the remaining code, but as a starting point ok
// in general I would do here a tokenizing of the fstrings to omit this peeking. // in general I would do here a tokenizing of the fstrings to omit this peeking.
'!' | '=' | '>' | '<' if self.peek() == Some(&'=') => { '!' | '=' | '>' | '<' if self.peek() == Some('=') => {
expression.push(ch); expression.push(ch);
expression.push('='); expression.push('=');
self.next_char(); self.next_char();
} }
'!' if delimiters.is_empty() && self.peek() != Some(&'=') => { '!' if delimiters.is_empty() && self.peek() != Some('=') => {
if expression.trim().is_empty() { if expression.trim().is_empty() {
return Err(FStringError::new(EmptyExpression, self.get_pos()).into()); return Err(FStringError::new(EmptyExpression, self.get_pos()).into());
} }
@ -231,18 +232,19 @@ impl<'a> StringParser<'a> {
// match a python 3.8 self documenting expression // match a python 3.8 self documenting expression
// format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}' // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}'
'=' if self.peek() != Some(&'=') && delimiters.is_empty() => { '=' if self.peek() != Some('=') && delimiters.is_empty() => {
self_documenting = true; self_documenting = true;
} }
':' if delimiters.is_empty() => { ':' if delimiters.is_empty() => {
let start_location = self.get_pos();
let parsed_spec = self.parse_spec(nested)?; let parsed_spec = self.parse_spec(nested)?;
spec = Some(Box::new( spec = Some(Box::new(
self.expr( self.expr(
ast::ExprJoinedStr { ast::ExprJoinedStr {
values: parsed_spec, values: parsed_spec,
range: self.range(), range: self.range(start_location),
} }
.into(), .into(),
), ),
@ -313,26 +315,30 @@ impl<'a> StringParser<'a> {
vec![self.expr( vec![self.expr(
ast::ExprFormattedValue { ast::ExprFormattedValue {
value: Box::new( value: Box::new(
parse_fstring_expr(&expression, location).map_err(|e| { parse_fstring_expr(&expression, start_location).map_err(
FStringError::new( |e| {
InvalidExpression(Box::new(e.error)), FStringError::new(
location, InvalidExpression(Box::new(e.error)),
) start_location,
})?, )
},
)?,
), ),
conversion, conversion,
format_spec: spec, format_spec: spec,
range: self.range(), range: self.range(start_location),
} }
.into(), .into(),
)] )]
} else { } else {
// TODO: range is wrong but `self_documenting` needs revisiting beyond
// ranges: https://github.com/astral-sh/ruff/issues/5970
vec![ vec![
self.expr( self.expr(
ast::ExprConstant { ast::ExprConstant {
value: Constant::Str(expression.to_owned() + "="), value: Constant::Str(expression.to_owned() + "="),
kind: None, kind: None,
range: self.range(), range: self.range(start_location),
} }
.into(), .into(),
), ),
@ -340,19 +346,21 @@ impl<'a> StringParser<'a> {
ast::ExprConstant { ast::ExprConstant {
value: trailing_seq.into(), value: trailing_seq.into(),
kind: None, kind: None,
range: self.range(), range: self.range(start_location),
} }
.into(), .into(),
), ),
self.expr( self.expr(
ast::ExprFormattedValue { ast::ExprFormattedValue {
value: Box::new( value: Box::new(
parse_fstring_expr(&expression, location).map_err(|e| { parse_fstring_expr(&expression, start_location).map_err(
FStringError::new( |e| {
InvalidExpression(Box::new(e.error)), FStringError::new(
location, InvalidExpression(Box::new(e.error)),
) start_location,
})?, )
},
)?,
), ),
conversion: if conversion == ConversionFlag::None conversion: if conversion == ConversionFlag::None
&& spec.is_none() && spec.is_none()
@ -362,7 +370,7 @@ impl<'a> StringParser<'a> {
conversion conversion
}, },
format_spec: spec, format_spec: spec,
range: self.range(), range: self.range(start_location),
} }
.into(), .into(),
), ),
@ -401,7 +409,8 @@ impl<'a> StringParser<'a> {
fn parse_spec(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> { fn parse_spec(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
let mut spec_constructor = Vec::new(); let mut spec_constructor = Vec::new();
let mut constant_piece = String::new(); let mut constant_piece = String::new();
while let Some(&next) = self.peek() { let mut start_location = self.get_pos();
while let Some(next) = self.peek() {
match next { match next {
'{' => { '{' => {
if !constant_piece.is_empty() { if !constant_piece.is_empty() {
@ -410,7 +419,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant { ast::ExprConstant {
value: constant_piece.drain(..).collect::<String>().into(), value: constant_piece.drain(..).collect::<String>().into(),
kind: None, kind: None,
range: self.range(), range: self.range(start_location),
} }
.into(), .into(),
), ),
@ -418,6 +427,7 @@ impl<'a> StringParser<'a> {
} }
let parsed_expr = self.parse_fstring(nested + 1)?; let parsed_expr = self.parse_fstring(nested + 1)?;
spec_constructor.extend(parsed_expr); spec_constructor.extend(parsed_expr);
start_location = self.get_pos();
continue; continue;
} }
'}' => { '}' => {
@ -435,7 +445,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant { ast::ExprConstant {
value: constant_piece.drain(..).collect::<String>().into(), value: constant_piece.drain(..).collect::<String>().into(),
kind: None, kind: None,
range: self.range(), range: self.range(start_location),
} }
.into(), .into(),
), ),
@ -452,15 +462,16 @@ impl<'a> StringParser<'a> {
} }
let mut content = String::new(); let mut content = String::new();
let mut start_location = self.get_pos();
let mut values = vec![]; let mut values = vec![];
while let Some(&ch) = self.peek() { while let Some(ch) = self.peek() {
match ch { match ch {
'{' => { '{' => {
self.next_char();
if nested == 0 { if nested == 0 {
match self.peek() { match self.peek2() {
Some('{') => { Some('{') => {
self.next_char();
self.next_char(); self.next_char();
content.push('{'); content.push('{');
continue; continue;
@ -477,7 +488,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant { ast::ExprConstant {
value: content.drain(..).collect::<String>().into(), value: content.drain(..).collect::<String>().into(),
kind: None, kind: None,
range: self.range(), range: self.range(start_location),
} }
.into(), .into(),
), ),
@ -486,6 +497,7 @@ impl<'a> StringParser<'a> {
let parsed_values = self.parse_formatted_value(nested)?; let parsed_values = self.parse_formatted_value(nested)?;
values.extend(parsed_values); values.extend(parsed_values);
start_location = self.get_pos();
} }
'}' => { '}' => {
if nested > 0 { if nested > 0 {
@ -516,7 +528,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant { ast::ExprConstant {
value: content.into(), value: content.into(),
kind: None, kind: None,
range: self.range(), range: self.range(start_location),
} }
.into(), .into(),
), ),
@ -528,6 +540,7 @@ impl<'a> StringParser<'a> {
fn parse_bytes(&mut self) -> Result<Expr, LexicalError> { fn parse_bytes(&mut self) -> Result<Expr, LexicalError> {
let mut content = String::new(); let mut content = String::new();
let start_location = self.get_pos();
while let Some(ch) = self.next_char() { while let Some(ch) = self.next_char() {
match ch { match ch {
'\\' if !self.kind.is_raw() => { '\\' if !self.kind.is_raw() => {
@ -551,7 +564,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant { ast::ExprConstant {
value: Constant::Bytes(content.chars().map(|c| c as u8).collect()), value: Constant::Bytes(content.chars().map(|c| c as u8).collect()),
kind: None, kind: None,
range: self.range(), range: self.range(start_location),
} }
.into(), .into(),
)) ))
@ -559,6 +572,7 @@ impl<'a> StringParser<'a> {
fn parse_string(&mut self) -> Result<Expr, LexicalError> { fn parse_string(&mut self) -> Result<Expr, LexicalError> {
let mut content = String::new(); let mut content = String::new();
let start_location = self.get_pos();
while let Some(ch) = self.next_char() { while let Some(ch) = self.next_char() {
match ch { match ch {
'\\' if !self.kind.is_raw() => { '\\' if !self.kind.is_raw() => {
@ -571,7 +585,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant { ast::ExprConstant {
value: Constant::Str(content), value: Constant::Str(content),
kind: self.kind.is_unicode().then(|| "u".to_string()), kind: self.kind.is_unicode().then(|| "u".to_string()),
range: self.range(), range: self.range(start_location),
} }
.into(), .into(),
)) ))
@ -590,8 +604,7 @@ impl<'a> StringParser<'a> {
fn parse_fstring_expr(source: &str, location: TextSize) -> Result<Expr, ParseError> { fn parse_fstring_expr(source: &str, location: TextSize) -> Result<Expr, ParseError> {
let fstring_body = format!("({source})"); let fstring_body = format!("({source})");
let start = location - TextSize::from(1); ast::Expr::parse_starts_at(&fstring_body, "<fstring>", location)
ast::Expr::parse_starts_at(&fstring_body, "<fstring>", start)
} }
fn parse_string( fn parse_string(
@ -599,9 +612,8 @@ fn parse_string(
kind: StringKind, kind: StringKind,
triple_quoted: bool, triple_quoted: bool,
start: TextSize, start: TextSize,
end: TextSize,
) -> Result<Vec<Expr>, LexicalError> { ) -> Result<Vec<Expr>, LexicalError> {
StringParser::new(source, kind, triple_quoted, start, end).parse() StringParser::new(source, kind, triple_quoted, start).parse()
} }
pub(crate) fn parse_strings( pub(crate) fn parse_strings(
@ -631,8 +643,8 @@ pub(crate) fn parse_strings(
if has_bytes { if has_bytes {
let mut content: Vec<u8> = vec![]; let mut content: Vec<u8> = vec![];
for (start, (source, kind, triple_quoted), end) in values { for (start, (source, kind, triple_quoted), _) in values {
for value in parse_string(&source, kind, triple_quoted, start, end)? { for value in parse_string(&source, kind, triple_quoted, start)? {
match value { match value {
Expr::Constant(ast::ExprConstant { Expr::Constant(ast::ExprConstant {
value: Constant::Bytes(value), value: Constant::Bytes(value),
@ -652,8 +664,8 @@ pub(crate) fn parse_strings(
if !has_fstring { if !has_fstring {
let mut content: Vec<String> = vec![]; let mut content: Vec<String> = vec![];
for (start, (source, kind, triple_quoted), end) in values { for (start, (source, kind, triple_quoted), _) in values {
for value in parse_string(&source, kind, triple_quoted, start, end)? { for value in parse_string(&source, kind, triple_quoted, start)? {
match value { match value {
Expr::Constant(ast::ExprConstant { Expr::Constant(ast::ExprConstant {
value: Constant::Str(value), value: Constant::Str(value),
@ -674,34 +686,43 @@ pub(crate) fn parse_strings(
// De-duplicate adjacent constants. // De-duplicate adjacent constants.
let mut deduped: Vec<Expr> = vec![]; let mut deduped: Vec<Expr> = vec![];
let mut current: Vec<String> = vec![]; let mut current: Vec<String> = vec![];
let mut current_start = initial_start;
let mut current_end = last_end;
let take_current = |current: &mut Vec<String>| -> Expr { let take_current = |current: &mut Vec<String>, start, end| -> Expr {
Expr::Constant(ast::ExprConstant { Expr::Constant(ast::ExprConstant {
value: Constant::Str(current.drain(..).join("")), value: Constant::Str(current.drain(..).join("")),
kind: initial_kind.clone(), kind: initial_kind.clone(),
range: TextRange::new(initial_start, last_end), range: TextRange::new(start, end),
}) })
}; };
for (start, (source, kind, triple_quoted), end) in values { for (start, (source, kind, triple_quoted), _) in values {
for value in parse_string(&source, kind, triple_quoted, start, end)? { for value in parse_string(&source, kind, triple_quoted, start)? {
let value_range = value.range();
match value { match value {
Expr::FormattedValue { .. } => { Expr::FormattedValue { .. } => {
if !current.is_empty() { if !current.is_empty() {
deduped.push(take_current(&mut current)); deduped.push(take_current(&mut current, current_start, current_end));
} }
deduped.push(value) deduped.push(value)
} }
Expr::Constant(ast::ExprConstant { Expr::Constant(ast::ExprConstant {
value: Constant::Str(value), value: Constant::Str(inner),
.. ..
}) => current.push(value), }) => {
if current.is_empty() {
current_start = value_range.start();
}
current_end = value_range.end();
current.push(inner);
}
_ => unreachable!("Unexpected non-string expression."), _ => unreachable!("Unexpected non-string expression."),
} }
} }
} }
if !current.is_empty() { if !current.is_empty() {
deduped.push(take_current(&mut current)); deduped.push(take_current(&mut current, current_start, current_end));
} }
Ok(Expr::JoinedStr(ast::ExprJoinedStr { Ok(Expr::JoinedStr(ast::ExprJoinedStr {
@ -818,14 +839,7 @@ mod tests {
use crate::{ast, Parse}; use crate::{ast, Parse};
fn parse_fstring(source: &str) -> Result<Vec<Expr>, LexicalError> { fn parse_fstring(source: &str) -> Result<Vec<Expr>, LexicalError> {
StringParser::new( StringParser::new(source, StringKind::FString, false, TextSize::default()).parse()
source,
StringKind::FString,
false,
TextSize::default(),
TextSize::default() + source.text_len() + TextSize::from(3), // 3 for prefix and quotes
)
.parse()
} }
#[test] #[test]
@ -1069,6 +1083,13 @@ mod tests {
insta::assert_debug_snapshot!(parse_ast); insta::assert_debug_snapshot!(parse_ast);
} }
#[test]
fn test_fstring_constant_range() {
let source = r#"f"aaa{bbb}ccc{ddd}eee""#;
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test] #[test]
fn test_fstring_unescaped_newline() { fn test_fstring_unescaped_newline() {
let source = r#"f""" let source = r#"f"""