Python/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff synced 2026-01-20 21:10:48 -05:00
Code Issues Packages Projects Releases Wiki Activity

Improved error recovery for unclosed strings (including f- and t-strings) (#20848)

Browse Source
This commit is contained in:
Micha Reiser
2025-10-15 09:50:56 +02:00
committed by GitHub
parent a93618ed23
commit 4fc7dd300c
151 changed files with 1370 additions and 566 deletions
Download Patch File Download Diff File Expand all files Collapse all files

314
crates/ruff_python_parser/src/lexer.rs
View File

@@ -13,6 +13,7 @@ use unicode_ident::{is_xid_continue, is_xid_start};
use unicode_normalization::UnicodeNormalization;
use ruff_python_ast::name::Name;
use ruff_python_ast::str_prefix::{AnyStringPrefix, StringLiteralPrefix};
use ruff_python_ast::{Int, IpyEscapeKind, StringFlags};
use ruff_python_trivia::is_python_whitespace;
use ruff_text_size::{TextLen, TextRange, TextSize};
@@ -24,6 +25,7 @@ use crate::lexer::indentation::{Indentation, Indentations, IndentationsCheckpoin
use crate::lexer::interpolated_string::{
InterpolatedStringContext, InterpolatedStrings, InterpolatedStringsCheckpoint,
};
use crate::string::InterpolatedStringKind;
use crate::token::{TokenFlags, TokenKind, TokenValue};
mod cursor;
@@ -782,6 +784,7 @@ impl<'src> Lexer<'src> {
// SAFETY: Safe because the function is only called when `self.fstrings` is not empty.
let interpolated_string = self.interpolated_strings.current().unwrap();
let string_kind = interpolated_string.kind();
let interpolated_flags = interpolated_string.flags();
// Check if we're at the end of the f-string.
if interpolated_string.is_triple_quoted() {
@@ -819,15 +822,19 @@ impl<'src> Lexer<'src> {
} else {
InterpolatedStringErrorType::UnterminatedString
};
self.nesting = interpolated_string.nesting();
self.interpolated_strings.pop();
return Some(self.push_error(LexicalError::new(
self.current_flags |= TokenFlags::UNCLOSED_STRING;
self.push_error(LexicalError::new(
LexicalErrorType::from_interpolated_string_error(error, string_kind),
self.token_range(),
)));
));
break;
}
'\n' | '\r' if !interpolated_string.is_triple_quoted() => {
// https://github.com/astral-sh/ruff/issues/18632
self.interpolated_strings.pop();
let error_type = if in_format_spec {
InterpolatedStringErrorType::NewlineInFormatSpec
@@ -835,10 +842,16 @@ impl<'src> Lexer<'src> {
InterpolatedStringErrorType::UnterminatedString
};
return Some(self.push_error(LexicalError::new(
self.nesting = interpolated_string.nesting();
self.interpolated_strings.pop();
self.current_flags |= TokenFlags::UNCLOSED_STRING;
self.push_error(LexicalError::new(
LexicalErrorType::from_interpolated_string_error(error_type, string_kind),
self.token_range(),
)));
));
break;
}
'\\' => {
self.cursor.bump(); // '\'
@@ -913,7 +926,7 @@ impl<'src> Lexer<'src> {
self.current_value = TokenValue::InterpolatedStringMiddle(value.into_boxed_str());
self.current_flags = interpolated_string.flags();
self.current_flags = interpolated_flags;
Some(string_kind.middle_token())
}
@@ -942,10 +955,12 @@ impl<'src> Lexer<'src> {
let Some(index) = memchr::memchr(quote_byte, self.cursor.rest().as_bytes()) else {
self.cursor.skip_to_end();
return self.push_error(LexicalError::new(
self.current_flags |= TokenFlags::UNCLOSED_STRING;
self.push_error(LexicalError::new(
LexicalErrorType::UnclosedStringError,
self.token_range(),
));
break self.offset();
};
// Rare case: if there are an odd number of backslashes before the quote, then
@@ -977,11 +992,14 @@ impl<'src> Lexer<'src> {
memchr::memchr3(quote_byte, b'\r', b'\n', self.cursor.rest().as_bytes())
else {
self.cursor.skip_to_end();
self.current_flags |= TokenFlags::UNCLOSED_STRING;
return self.push_error(LexicalError::new(
LexicalErrorType::StringError,
self.push_error(LexicalError::new(
LexicalErrorType::UnclosedStringError,
self.token_range(),
));
break self.offset();
};
// Rare case: if there are an odd number of backslashes before the quote, then
@@ -1009,10 +1027,12 @@ impl<'src> Lexer<'src> {
match quote_or_newline {
'\r' | '\n' => {
return self.push_error(LexicalError::new(
self.current_flags |= TokenFlags::UNCLOSED_STRING;
self.push_error(LexicalError::new(
LexicalErrorType::UnclosedStringError,
self.token_range(),
));
break self.offset();
}
ch if ch == quote => {
let value_end = self.offset();
@@ -1331,7 +1351,20 @@ impl<'src> Lexer<'src> {
fn consume_end(&mut self) -> TokenKind {
// We reached end of file.
// First of all, we need all nestings to be finished.
// First, finish any unterminated interpolated-strings.
while let Some(interpolated_string) = self.interpolated_strings.pop() {
self.nesting = interpolated_string.nesting();
self.push_error(LexicalError::new(
LexicalErrorType::from_interpolated_string_error(
InterpolatedStringErrorType::UnterminatedString,
interpolated_string.kind(),
),
self.token_range(),
));
}
// Second, finish all nestings.
// For Mode::ParenthesizedExpression we start with nesting level 1.
// So we check if we end with that level.
let init_nesting = u32::from(self.mode == Mode::ParenthesizedExpression);
@@ -1459,6 +1492,107 @@ impl<'src> Lexer<'src> {
true
}
/// Re-lexes an unclosed string token in the context of an interpolated string element.
///
/// ```py
/// f'{a'
/// ```
///
/// This method re-lexes the trailing `'` as the end of the f-string rather than the
/// start of a new string token for better error recovery.
pub(crate) fn re_lex_string_token_in_interpolation_element(
&mut self,
kind: InterpolatedStringKind,
) {
let Some(interpolated_string) = self.interpolated_strings.current() else {
return;
};
let current_string_flags = self.current_flags().as_any_string_flags();
// Only unclosed strings, that have the same quote character
if !matches!(self.current_kind, TokenKind::String)
|| !self.current_flags.is_unclosed()
|| current_string_flags.prefix() != AnyStringPrefix::Regular(StringLiteralPrefix::Empty)
|| current_string_flags.quote_style().as_char() != interpolated_string.quote_char()
|| current_string_flags.is_triple_quoted() != interpolated_string.is_triple_quoted()
{
return;
}
// Only if the string's first line only contains whitespace,
// or ends in a comment (not `f"{"abc`)
let first_line = &self.source
[(self.current_range.start() + current_string_flags.quote_len()).to_usize()..];
for c in first_line.chars() {
if matches!(c, '\n' | '\r' | '#') {
break;
}
// `f'{'ab`, we want to parse `ab` as a normal string and not the closing element of the f-string
if !is_python_whitespace(c) {
return;
}
}
if self.errors.last().is_some_and(|error| {
error.location() == self.current_range
&& matches!(error.error(), LexicalErrorType::UnclosedStringError)
}) {
self.errors.pop();
}
self.current_range =
TextRange::at(self.current_range.start(), self.current_flags.quote_len());
self.current_kind = kind.end_token();
self.current_value = TokenValue::None;
self.current_flags = TokenFlags::empty();
self.nesting = interpolated_string.nesting();
self.interpolated_strings.pop();
self.cursor = Cursor::new(self.source);
self.cursor.skip_bytes(self.current_range.end().to_usize());
}
/// Re-lex `r"` in a format specifier position.
///
/// `r"` in a format specifier position is unlikely to be the start of a raw string.
/// Instead, it's the format specifier `!r` followed by the closing quote of the f-string,
/// when the `}` is missing.
///
/// ```py
/// f"{test!r"
/// ```
///
/// This function re-lexes the `r"` as `r` (a name token). The next `next_token` call will
/// return a unclosed string token for `"`, which [`Self::re_lex_string_token_in_interpolation_element`]
/// can then re-lex as the end of the f-string.
pub(crate) fn re_lex_raw_string_in_format_spec(&mut self) {
// Re-lex `r"` as `NAME r` followed by an unclosed string
// `f"{test!r"` -> `f"{test!`, `r`, `"`
if matches!(self.current_kind, TokenKind::String)
&& self.current_flags.is_unclosed()
&& self.current_flags.prefix()
== AnyStringPrefix::Regular(StringLiteralPrefix::Raw { uppercase: false })
{
if self.errors.last().is_some_and(|error| {
error.location() == self.current_range
&& matches!(error.error(), LexicalErrorType::UnclosedStringError)
}) {
self.errors.pop();
}
self.current_range = TextRange::at(self.current_range.start(), 'r'.text_len());
self.current_kind = TokenKind::Name;
self.current_value = TokenValue::Name(Name::new_static("r"));
self.current_flags = TokenFlags::empty();
self.cursor = Cursor::new(self.source);
self.cursor.skip_bytes(self.current_range.end().to_usize());
}
}
#[inline]
fn token_range(&self) -> TextRange {
let end = self.offset();
@@ -2739,6 +2873,164 @@ t"{(lambda x:{x})}"
}
}
#[test]
fn lex_fstring_unclosed() {
let source = r#"f"hello"#;
assert_snapshot!(lex_invalid(source, Mode::Module), @r#"
## Tokens
```
[
(
FStringStart,
0..2,
TokenFlags(
DOUBLE_QUOTES | F_STRING,
),
),
(
InterpolatedStringMiddle(
"hello",
),
2..7,
TokenFlags(
DOUBLE_QUOTES | F_STRING,
),
),
(
Newline,
7..7,
),
]
```
## Errors
```
[
LexicalError {
error: FStringError(
UnterminatedString,
),
location: 2..7,
},
]
```
"#);
}
#[test]
fn lex_fstring_missing_brace() {
let source = "f'{'";
assert_snapshot!(lex_invalid(source, Mode::Module), @r#"
## Tokens
```
[
(
FStringStart,
0..2,
TokenFlags(
F_STRING,
),
),
(
Lbrace,
2..3,
),
(
String(
"",
),
3..4,
TokenFlags(
UNCLOSED_STRING,
),
),
(
Newline,
4..4,
),
]
```
## Errors
```
[
LexicalError {
error: UnclosedStringError,
location: 3..4,
},
LexicalError {
error: FStringError(
UnterminatedString,
),
location: 4..4,
},
]
```
"#);
}
#[test]
fn lex_fstring_missing_brace_after_format_spec() {
let source = r#"f"{foo!r""#;
assert_snapshot!(lex_invalid(source, Mode::Module), @r#"
## Tokens
```
[
(
FStringStart,
0..2,
TokenFlags(
DOUBLE_QUOTES | F_STRING,
),
),
(
Lbrace,
2..3,
),
(
Name(
Name("foo"),
),
3..6,
),
(
Exclamation,
6..7,
),
(
String(
"",
),
7..9,
TokenFlags(
DOUBLE_QUOTES | RAW_STRING_LOWERCASE | UNCLOSED_STRING,
),
),
(
Newline,
9..9,
),
]
```
## Errors
```
[
LexicalError {
error: UnclosedStringError,
location: 7..9,
},
LexicalError {
error: FStringError(
UnterminatedString,
),
location: 9..9,
},
]
```
"#);
}
#[test]
fn test_tstring_error() {
use InterpolatedStringErrorType::{

17
crates/ruff_python_parser/src/parser/expression.rs
View File

@@ -1526,7 +1526,7 @@ impl<'src> Parser<'src> {
kind: InterpolatedStringKind,
) -> InterpolatedStringData {
let start = self.node_start();
let flags = self.tokens.current_flags().as_any_string_flags();
let mut flags = self.tokens.current_flags().as_any_string_flags();
self.bump(kind.start_token());
let elements = self.parse_interpolated_string_elements(
@@ -1535,7 +1535,9 @@ impl<'src> Parser<'src> {
kind,
);
self.expect(kind.end_token());
if !self.expect(kind.end_token()) {
flags = flags.with_unclosed(true);
}
// test_ok pep701_f_string_py312
// # parse_options: {"target-version": "3.12"}
@@ -1719,6 +1721,9 @@ impl<'src> Parser<'src> {
let start = self.node_start();
self.bump(TokenKind::Lbrace);
self.tokens
.re_lex_string_token_in_interpolation_element(string_kind);
// test_err f_string_empty_expression
// f"{}"
// f"{ }"
@@ -1740,6 +1745,7 @@ impl<'src> Parser<'src> {
// t"{*}"
// t"{*x and y}"
// t"{*yield x}"
let value = self.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or());
if !value.is_parenthesized && value.expr.is_lambda_expr() {
@@ -1773,6 +1779,10 @@ impl<'src> Parser<'src> {
};
let conversion = if self.eat(TokenKind::Exclamation) {
// Ensure that the `r` is lexed as a `r` name token instead of a raw string
// in `f{abc!r"` (note the missing `}`).
self.tokens.re_lex_raw_string_in_format_spec();
let conversion_flag_range = self.current_token_range();
if self.at(TokenKind::Name) {
// test_err f_string_conversion_follows_exclamation
@@ -1852,6 +1862,9 @@ impl<'src> Parser<'src> {
None
};
self.tokens
.re_lex_string_token_in_interpolation_element(string_kind);
// We're using `eat` here instead of `expect` to use the f-string specific error type.
if !self.eat(TokenKind::Rbrace) {
// TODO(dhruvmanila): This requires some changes in the lexer. One of them

1
crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__unicode_aliases.snap
View File

@@ -31,6 +31,7 @@ expression: suite
quote_style: Double,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

42
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_newline_format_spec.snap
View File

@@ -44,11 +44,16 @@ expression: "lex_invalid(source, Mode::Module)"
12..13,
),
(
Unknown,
InterpolatedStringMiddle(
"d",
),
13..14,
TokenFlags(
F_STRING,
),
),
(
NonLogicalNewline,
Newline,
14..15,
),
(
@@ -62,8 +67,13 @@ expression: "lex_invalid(source, Mode::Module)"
16..18,
),
(
Unknown,
String(
"",
),
18..19,
TokenFlags(
UNCLOSED_STRING,
),
),
(
Newline,
@@ -104,13 +114,22 @@ expression: "lex_invalid(source, Mode::Module)"
31..32,
),
(
Unknown,
InterpolatedStringMiddle(
"a",
),
32..33,
TokenFlags(
F_STRING,
),
),
(
NonLogicalNewline,
Newline,
33..34,
),
(
Indent,
34..42,
),
(
Name(
Name("b"),
@@ -118,9 +137,13 @@ expression: "lex_invalid(source, Mode::Module)"
42..43,
),
(
NonLogicalNewline,
Newline,
43..44,
),
(
Dedent,
44..44,
),
(
Rbrace,
44..45,
@@ -132,8 +155,13 @@ expression: "lex_invalid(source, Mode::Module)"
45..47,
),
(
Unknown,
String(
"",
),
47..48,
TokenFlags(
UNCLOSED_STRING,
),
),
(
Newline,

42
crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__tstring_newline_format_spec.snap
View File

@@ -44,11 +44,16 @@ expression: "lex_invalid(source, Mode::Module)"
12..13,
),
(
Unknown,
InterpolatedStringMiddle(
"d",
),
13..14,
TokenFlags(
T_STRING,
),
),
(
NonLogicalNewline,
Newline,
14..15,
),
(
@@ -62,8 +67,13 @@ expression: "lex_invalid(source, Mode::Module)"
16..18,
),
(
Unknown,
String(
"",
),
18..19,
TokenFlags(
UNCLOSED_STRING,
),
),
(
Newline,
@@ -104,13 +114,22 @@ expression: "lex_invalid(source, Mode::Module)"
31..32,
),
(
Unknown,
InterpolatedStringMiddle(
"a",
),
32..33,
TokenFlags(
T_STRING,
),
),
(
NonLogicalNewline,
Newline,
33..34,
),
(
Indent,
34..42,
),
(
Name(
Name("b"),
@@ -118,9 +137,13 @@ expression: "lex_invalid(source, Mode::Module)"
42..43,
),
(
NonLogicalNewline,
Newline,
43..44,
),
(
Dedent,
44..44,
),
(
Rbrace,
44..45,
@@ -132,8 +155,13 @@ expression: "lex_invalid(source, Mode::Module)"
45..47,
),
(
Unknown,
String(
"",
),
47..48,
TokenFlags(
UNCLOSED_STRING,
),
),
(
Newline,

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__backspace_alias.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Double,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__bell_alias.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Double,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__carriage_return_alias.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Double,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__character_tabulation_with_justification_alias.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Double,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__delete_alias.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Double,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__dont_panic_on_8_in_octal_escape.snap
View File

@@ -31,6 +31,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__double_quoted_byte.snap
View File

@@ -278,6 +278,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__escape_alias.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Double,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__escape_char_in_byte_literal.snap
View File

@@ -32,6 +32,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__escape_octet.snap
View File

@@ -27,6 +27,7 @@ expression: suite
quote_style: Single,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__form_feed_alias.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Double,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__fstring_constant_range.snap
View File

@@ -78,6 +78,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__fstring_escaped_character.snap
View File

@@ -47,6 +47,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__fstring_escaped_newline.snap
View File

@@ -47,6 +47,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__fstring_line_continuation.snap
View File

@@ -49,6 +49,7 @@ expression: suite
uppercase_r: false,
},
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__fstring_parse_self_documenting_base.snap
View File

@@ -45,6 +45,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__fstring_parse_self_documenting_base_more.snap
View File

@@ -81,6 +81,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__fstring_parse_self_documenting_format.snap
View File

@@ -59,6 +59,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__fstring_unescaped_newline.snap
View File

@@ -47,6 +47,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: true,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__hts_alias.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Double,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_empty_fstring.snap
View File

@@ -22,6 +22,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_empty_tstring.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_f_string_concat_1.snap
View File

@@ -23,6 +23,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),
@@ -43,6 +44,7 @@ expression: suite
quote_style: Single,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_f_string_concat_2.snap
View File

@@ -23,6 +23,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),
@@ -43,6 +44,7 @@ expression: suite
quote_style: Single,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

3
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_f_string_concat_3.snap
View File

@@ -23,6 +23,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),
@@ -56,6 +57,7 @@ expression: suite
quote_style: Double,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),
@@ -72,6 +74,7 @@ expression: suite
quote_style: Single,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

4
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_f_string_concat_4.snap
View File

@@ -23,6 +23,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),
@@ -56,6 +57,7 @@ expression: suite
quote_style: Double,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),
@@ -72,6 +74,7 @@ expression: suite
quote_style: Single,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),
@@ -84,6 +87,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring.snap
View File

@@ -64,6 +64,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_equals.snap
View File

@@ -61,6 +61,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

3
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_nested_concatenation_string_spec.snap
View File

@@ -57,6 +57,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
StringLiteral {
@@ -67,6 +68,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
],
@@ -91,6 +93,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_nested_spec.snap
View File

@@ -64,6 +64,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_nested_string_spec.snap
View File

@@ -55,6 +55,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),
@@ -76,6 +77,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_not_equals.snap
View File

@@ -61,6 +61,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_not_nested_spec.snap
View File

@@ -54,6 +54,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_self_doc_prec_space.snap
View File

@@ -45,6 +45,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_self_doc_trailing_space.snap
View File

@@ -45,6 +45,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_fstring_yield_expr.snap
View File

@@ -39,6 +39,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_string_concat.snap
View File

@@ -23,6 +23,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
StringLiteral {
@@ -33,6 +34,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
],

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_string_triple_quotes_with_kind.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Single,
prefix: Unicode,
triple_quoted: true,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring.snap
View File

@@ -63,6 +63,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_equals.snap
View File

@@ -60,6 +60,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

3
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_nested_concatenation_string_spec.snap
View File

@@ -56,6 +56,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
StringLiteral {
@@ -66,6 +67,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
],
@@ -90,6 +92,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_nested_spec.snap
View File

@@ -63,6 +63,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_nested_string_spec.snap
View File

@@ -54,6 +54,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),
@@ -75,6 +76,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_not_equals.snap
View File

@@ -60,6 +60,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_not_nested_spec.snap
View File

@@ -53,6 +53,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_self_doc_prec_space.snap
View File

@@ -44,6 +44,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_self_doc_trailing_space.snap
View File

@@ -44,6 +44,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_tstring_yield_expr.snap
View File

@@ -38,6 +38,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_u_f_string_concat_1.snap
View File

@@ -23,6 +23,7 @@ expression: suite
quote_style: Single,
prefix: Unicode,
triple_quoted: false,
unclosed: false,
},
},
),
@@ -43,6 +44,7 @@ expression: suite
quote_style: Single,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

3
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_u_f_string_concat_2.snap
View File

@@ -23,6 +23,7 @@ expression: suite
quote_style: Single,
prefix: Unicode,
triple_quoted: false,
unclosed: false,
},
},
),
@@ -43,6 +44,7 @@ expression: suite
quote_style: Single,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),
@@ -55,6 +57,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_u_string_concat_1.snap
View File

@@ -23,6 +23,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
StringLiteral {
@@ -33,6 +34,7 @@ expression: suite
quote_style: Single,
prefix: Unicode,
triple_quoted: false,
unclosed: false,
},
},
],

2
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__parse_u_string_concat_2.snap
View File

@@ -23,6 +23,7 @@ expression: suite
quote_style: Single,
prefix: Unicode,
triple_quoted: false,
unclosed: false,
},
},
StringLiteral {
@@ -33,6 +34,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
],

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__raw_byte_literal_1.snap
View File

@@ -28,6 +28,7 @@ expression: suite
uppercase_r: false,
},
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__raw_byte_literal_2.snap
View File

@@ -26,6 +26,7 @@ expression: suite
uppercase_r: false,
},
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__raw_fstring.snap
View File

@@ -42,6 +42,7 @@ expression: suite
uppercase_r: false,
},
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__raw_tstring.snap
View File

@@ -41,6 +41,7 @@ expression: suite
uppercase_r: false,
},
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__single_quoted_byte.snap
View File

@@ -278,6 +278,7 @@ expression: suite
quote_style: Single,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__string_parser_escaped_mac_eol.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__string_parser_escaped_unix_eol.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__string_parser_escaped_windows_eol.snap
View File

@@ -21,6 +21,7 @@ expression: suite
quote_style: Single,
prefix: Empty,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__triple_quoted_raw_fstring.snap
View File

@@ -42,6 +42,7 @@ expression: suite
uppercase_r: false,
},
triple_quoted: true,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__triple_quoted_raw_tstring.snap
View File

@@ -41,6 +41,7 @@ expression: suite
uppercase_r: false,
},
triple_quoted: true,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__tstring_constant_range.snap
View File

@@ -77,6 +77,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__tstring_escaped_character.snap
View File

@@ -46,6 +46,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__tstring_escaped_newline.snap
View File

@@ -46,6 +46,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__tstring_line_continuation.snap
View File

@@ -48,6 +48,7 @@ expression: suite
uppercase_r: false,
},
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__tstring_parse_self_documenting_base.snap
View File

@@ -44,6 +44,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__tstring_parse_self_documenting_base_more.snap
View File

@@ -80,6 +80,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__tstring_parse_self_documenting_format.snap
View File

@@ -58,6 +58,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: false,
unclosed: false,
},
},
),

1
crates/ruff_python_parser/src/snapshots/ruff_python_parser__string__tests__tstring_unescaped_newline.snap
View File

@@ -46,6 +46,7 @@ expression: suite
quote_style: Double,
prefix: Regular,
triple_quoted: true,
unclosed: false,
},
},
),

9
crates/ruff_python_parser/src/token.rs
View File

@@ -729,7 +729,7 @@ impl fmt::Display for TokenKind {
bitflags! {
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct TokenFlags: u8 {
pub(crate) struct TokenFlags: u16 {
/// The token is a string with double quotes (`"`).
const DOUBLE_QUOTES = 1 << 0;
/// The token is a triple-quoted string i.e., it starts and ends with three consecutive
@@ -748,9 +748,12 @@ bitflags! {
const RAW_STRING_LOWERCASE = 1 << 6;
/// The token is a raw string and the prefix character is in uppercase.
const RAW_STRING_UPPERCASE = 1 << 7;
/// String without matching closing quote(s)
const UNCLOSED_STRING = 1 << 8;
/// The token is a raw string i.e., prefixed with `r` or `R`
const RAW_STRING = Self::RAW_STRING_LOWERCASE.bits() | Self::RAW_STRING_UPPERCASE.bits();
}
}
@@ -808,6 +811,10 @@ impl StringFlags for TokenFlags {
AnyStringPrefix::Regular(StringLiteralPrefix::Empty)
}
}
fn is_unclosed(self) -> bool {
self.intersects(TokenFlags::UNCLOSED_STRING)
}
}
impl TokenFlags {

13
crates/ruff_python_parser/src/token_source.rs
View File

@@ -3,6 +3,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize};
use crate::Mode;
use crate::error::LexicalError;
use crate::lexer::{Lexer, LexerCheckpoint};
use crate::string::InterpolatedStringKind;
use crate::token::{Token, TokenFlags, TokenKind, TokenValue};
/// Token source for the parser that skips over any trivia tokens.
@@ -88,6 +89,18 @@ impl<'src> TokenSource<'src> {
}
}
pub(crate) fn re_lex_string_token_in_interpolation_element(
&mut self,
kind: InterpolatedStringKind,
) {
self.lexer
.re_lex_string_token_in_interpolation_element(kind);
}
pub(crate) fn re_lex_raw_string_in_format_spec(&mut self) {
self.lexer.re_lex_raw_string_in_format_spec();
}
/// Returns the next non-trivia token without consuming it.
///
/// Use [`peek2`] to get the next two tokens.