From c4e32ea1801d8bff0c84a113bdaa3a9acd11de65 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Wed, 10 Dec 2025 17:16:28 +0100 Subject: [PATCH] Sync interpolated lexer state within `re_lex_logical_token` --- crates/ruff_python_ast/src/token.rs | 2 +- crates/ruff_python_parser/src/lexer.rs | 16 +++++-- .../invalid_syntax@re_lexing__ty_1828.py.snap | 44 +++++++++---------- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/crates/ruff_python_ast/src/token.rs b/crates/ruff_python_ast/src/token.rs index 4b9d98ec5c..0dc09a0f56 100644 --- a/crates/ruff_python_ast/src/token.rs +++ b/crates/ruff_python_ast/src/token.rs @@ -841,7 +841,7 @@ impl TokenFlags { self.intersects(TokenFlags::T_STRING.union(TokenFlags::F_STRING)) } - /// Returns `true` if the token is a triple-quoted t-string. + /// Returns `true` if the token is a triple-quoted interpolated-string. pub fn is_triple_quoted_interpolated_string(self) -> bool { self.intersects(TokenFlags::TRIPLE_QUOTED_STRING) && self.is_interpolated_string() } diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs index 8b4b3a061c..7d831aa6f5 100644 --- a/crates/ruff_python_parser/src/lexer.rs +++ b/crates/ruff_python_parser/src/lexer.rs @@ -1452,14 +1452,20 @@ impl<'src> Lexer<'src> { return false; } + if let Some(interpolated) = self.interpolated_strings.current_mut() { + interpolated.try_end_format_spec(self.nesting); + } + // Reduce the nesting level because the parser recovered from an error inside list parsing // i.e., it recovered from an unclosed parenthesis (`(`, `[`, or `{`). self.nesting -= 1; - // The lexer can't be moved back for a triple-quoted f/t-string because the newlines are - // part of the f/t-string itself, so there is no newline token to be emitted. - if self.current_flags.is_triple_quoted_interpolated_string() { - return false; + // A nesting level that's lower than the nesting when the interpolated string was created + // strongly suggests that we're now outside an interpolated string. + if let Some(interpolated) = self.interpolated_strings.current() + && interpolated.nesting() < self.nesting + { + self.interpolated_strings.pop(); } let Some(new_position) = non_logical_newline_start else { @@ -1537,6 +1543,8 @@ impl<'src> Lexer<'src> { } } + dbg!("Handle unclosed string"); + if self.errors.last().is_some_and(|error| { error.location() == self.current_range && matches!(error.error(), LexicalErrorType::UnclosedStringError) diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__ty_1828.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__ty_1828.py.snap index 49ea0c7d58..19966d08d9 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__ty_1828.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__ty_1828.py.snap @@ -145,6 +145,28 @@ Module( simple: false, }, ), + ClassDef( + StmtClassDef { + node_index: NodeIndex(None), + range: 94..111, + decorator_list: [], + name: Identifier { + id: Name("A"), + range: 100..101, + node_index: NodeIndex(None), + }, + type_params: None, + arguments: None, + body: [ + Pass( + StmtPass { + node_index: NodeIndex(None), + range: 107..111, + }, + ), + ], + }, + ), ], }, ) @@ -189,17 +211,6 @@ Module( | - | -1 | # Regression test for https://github.com/astral-sh/ty/issues/1828 -2 | (c: int = 1,f"""{d=[ -3 | def a( - | _______^ -4 | | class A: -5 | | pass - | |_________^ Syntax Error: f-string: unterminated triple-quoted string - | - - | 2 | (c: int = 1,f"""{d=[ 3 | def a( @@ -209,17 +220,6 @@ Module( | - | -1 | # Regression test for https://github.com/astral-sh/ty/issues/1828 -2 | (c: int = 1,f"""{d=[ -3 | def a( - | _______^ -4 | | class A: -5 | | pass - | |_________^ Syntax Error: Expected a statement - | - - | 4 | class A: 5 | pass