diff --git a/Cargo.lock b/Cargo.lock index 8b8f9fec2f..782eff98a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2190,7 +2190,6 @@ dependencies = [ "anyhow", "bitflags 2.5.0", "bstr", - "drop_bomb", "insta", "is-macro", "itertools 0.12.1", diff --git a/crates/ruff_python_parser/Cargo.toml b/crates/ruff_python_parser/Cargo.toml index 319df3eb0e..1422d9f980 100644 --- a/crates/ruff_python_parser/Cargo.toml +++ b/crates/ruff_python_parser/Cargo.toml @@ -18,7 +18,6 @@ ruff_text_size = { path = "../ruff_text_size" } anyhow = { workspace = true } bitflags = { workspace = true } -drop_bomb = { workspace = true } bstr = { workspace = true } is-macro = { workspace = true } itertools = { workspace = true } diff --git a/crates/ruff_python_parser/resources/inline/err/for_in_target_postfix_expr.py b/crates/ruff_python_parser/resources/inline/err/for_in_target_postfix_expr.py deleted file mode 100644 index 6b20f4f0fc..0000000000 --- a/crates/ruff_python_parser/resources/inline/err/for_in_target_postfix_expr.py +++ /dev/null @@ -1 +0,0 @@ -for d(x in y) in target: ... diff --git a/crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target.py b/crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target.py index d6de1bdc06..3b3e6563e4 100644 --- a/crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target.py +++ b/crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target.py @@ -3,4 +3,5 @@ for "a" in x: ... for *x and y in z: ... for *x | y in z: ... for await x in z: ... +for yield x in y: ... for [x, 1, y, *["a"]] in z: ... diff --git a/crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target_binary_expr.py b/crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target_binary_expr.py new file mode 100644 index 0000000000..fe4eb84631 --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target_binary_expr.py @@ -0,0 +1,6 @@ +for x not in y in z: ... +for x == y in z: ... +for x or y in z: ... +for -x in y: ... +for not x in y: ... +for x | y in z: ... diff --git a/crates/ruff_python_parser/resources/inline/err/parenthesized_compare_expr_in_for.py b/crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target_in_keyword.py similarity index 82% rename from crates/ruff_python_parser/resources/inline/err/parenthesized_compare_expr_in_for.py rename to crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target_in_keyword.py index fd06f2e422..80d895d294 100644 --- a/crates/ruff_python_parser/resources/inline/err/parenthesized_compare_expr_in_for.py +++ b/crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target_in_keyword.py @@ -1,3 +1,4 @@ +for d(x in y) in target: ... for (x in y)() in iter: ... for (x in y) in iter: ... for (x in y, z) in iter: ... diff --git a/crates/ruff_python_parser/resources/inline/ok/for_in_target_postfix_expr.py b/crates/ruff_python_parser/resources/inline/ok/for_in_target_postfix_expr.py deleted file mode 100644 index bf905dbdc8..0000000000 --- a/crates/ruff_python_parser/resources/inline/ok/for_in_target_postfix_expr.py +++ /dev/null @@ -1 +0,0 @@ -for d[x in y] in target: ... diff --git a/crates/ruff_python_parser/resources/inline/ok/parenthesized_compare_expr_in_for.py b/crates/ruff_python_parser/resources/inline/ok/for_in_target_valid_expr.py similarity index 67% rename from crates/ruff_python_parser/resources/inline/ok/parenthesized_compare_expr_in_for.py rename to crates/ruff_python_parser/resources/inline/ok/for_in_target_valid_expr.py index ba2b3c2cfa..49dd9df728 100644 --- a/crates/ruff_python_parser/resources/inline/ok/parenthesized_compare_expr_in_for.py +++ b/crates/ruff_python_parser/resources/inline/ok/for_in_target_valid_expr.py @@ -1,2 +1,3 @@ +for d[x in y] in target: ... for (x in y)[0] in iter: ... for (x in y).attr in iter: ... diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 8683c27643..5d2f492734 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -2,6 +2,7 @@ use std::cmp::Ordering; use std::hash::BuildHasherDefault; use std::ops::Deref; +use bitflags::bitflags; use rustc_hash::FxHashSet; use ruff_python_ast::{ @@ -12,7 +13,7 @@ use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use crate::parser::helpers::token_kind_to_cmp_op; use crate::parser::progress::ParserProgress; -use crate::parser::{helpers, FunctionKind, Parser, ParserCtxFlags}; +use crate::parser::{helpers, FunctionKind, Parser}; use crate::string::{parse_fstring_literal_element, parse_string_literal, StringType}; use crate::token_set::TokenSet; use crate::{FStringErrorType, Mode, ParseErrorType, Tok, TokenKind}; @@ -118,29 +119,20 @@ impl<'src> Parser<'src> { /// Parses every Python expression. /// - /// Matches the `expressions` rule in the [Python grammar]. - /// - /// The caller can specify whether starred expression is allowed or not. This - /// doesn't affect the parsing of a starred expression as it will be parsed - /// nevertheless. But, if it is not allowed, an error is reported. - /// - /// Use [`Parser::parse_star_expression_list`] if the starred expression is - /// required with a bitwise OR precedence. + /// Matches the `expressions` rule in the [Python grammar]. The [`ExpressionContext`] can be + /// used to match the `star_expressions` rule. /// /// [Python grammar]: https://docs.python.org/3/reference/grammar.html - pub(super) fn parse_expression_list( - &mut self, - allow_starred_expression: AllowStarredExpression, - ) -> ParsedExpr { + pub(super) fn parse_expression_list(&mut self, context: ExpressionContext) -> ParsedExpr { let start = self.node_start(); - let parsed_expr = self.parse_conditional_expression_or_higher(allow_starred_expression); + let parsed_expr = self.parse_conditional_expression_or_higher_impl(context); if self.at(TokenKind::Comma) { Expr::Tuple(self.parse_tuple_expression( parsed_expr.expr, start, Parenthesized::No, - |p| p.parse_conditional_expression_or_higher(allow_starred_expression), + |p| p.parse_conditional_expression_or_higher_impl(context), )) .into() } else { @@ -148,76 +140,21 @@ impl<'src> Parser<'src> { } } - /// Parses every Python expression. - /// - /// Matches the `star_expressions` rule in the [Python grammar]. - /// - /// [Python grammar]: https://docs.python.org/3/reference/grammar.html - pub(super) fn parse_star_expression_list(&mut self) -> ParsedExpr { - let start = self.node_start(); - let parsed_expr = self.parse_star_expression_or_higher(AllowNamedExpression::No); - - if self.at(TokenKind::Comma) { - Expr::Tuple(self.parse_tuple_expression( - parsed_expr.expr, - start, - Parenthesized::No, - |parser| parser.parse_star_expression_or_higher(AllowNamedExpression::No), - )) - .into() - } else { - parsed_expr - } - } - - /// Parses a star expression or any other expression. - /// - /// Matches either the `star_named_expression` or `star_expression` rule in - /// the [Python grammar] depending on whether named expressions are allowed - /// or not respectively. - /// - /// NOTE: If you have expressions separated by commas and want to parse them - /// individually instead of as a tuple, as done by [`Parser::parse_star_expression_list`], - /// use this function. - /// - /// [Python grammar]: https://docs.python.org/3/reference/grammar.html - pub(super) fn parse_star_expression_or_higher( - &mut self, - allow_named_expression: AllowNamedExpression, - ) -> ParsedExpr { - // This method parses starred expression with a different precedence, - // so don't allow starred expression in other branches. - if self.at(TokenKind::Star) { - Expr::Starred(self.parse_starred_expression(StarredExpressionPrecedence::BitOr)).into() - } else if allow_named_expression.is_yes() { - self.parse_named_expression_or_higher(AllowStarredExpression::No) - } else { - self.parse_conditional_expression_or_higher(AllowStarredExpression::No) - } - } - /// Parses every Python expression except unparenthesized tuple. /// - /// Matches the `named_expression` rule in the [Python grammar]. + /// Matches the `named_expression` rule in the [Python grammar]. The [`ExpressionContext`] can + /// be used to match the `star_named_expression` rule. /// - /// The caller can specify whether starred expression is allowed or not. This - /// doesn't affect the parsing of a starred expression as it will be parsed - /// nevertheless. But, if it is not allowed, an error is reported. - /// - /// Use [`Parser::parse_star_expression_or_higher`] with [`AllowNamedExpression::Yes`] - /// if the starred expression is required with a bitwise OR precedence. - /// - /// NOTE: If you have expressions separated by commas and want to parse them - /// individually instead of as a tuple, as done by [`Parser::parse_expression_list`] - /// use this function! + /// NOTE: If you have expressions separated by commas and want to parse them individually + /// instead of as a tuple, as done by [`Parser::parse_expression_list`], use this function. /// /// [Python grammar]: https://docs.python.org/3/reference/grammar.html pub(super) fn parse_named_expression_or_higher( &mut self, - allow_starred_expression: AllowStarredExpression, + context: ExpressionContext, ) -> ParsedExpr { let start = self.node_start(); - let parsed_expr = self.parse_conditional_expression_or_higher(allow_starred_expression); + let parsed_expr = self.parse_conditional_expression_or_higher_impl(context); if self.at(TokenKind::ColonEqual) { Expr::Named(self.parse_named_expression(parsed_expr.expr, start)).into() @@ -230,27 +167,27 @@ impl<'src> Parser<'src> { /// /// Matches the `expression` rule in the [Python grammar]. /// - /// The caller can specify whether starred expression is allowed or not. This - /// doesn't affect the parsing of a starred expression as it will be parsed - /// nevertheless. But, if it is not allowed, an error is reported. + /// This uses the default [`ExpressionContext`]. Use + /// [`Parser::parse_conditional_expression_or_higher_impl`] if you prefer to pass in the + /// context. /// - /// Use [`Parser::parse_star_expression_or_higher`] with [`AllowNamedExpression::No`] - /// if the starred expression is required with a bitwise OR precedence. - /// - /// NOTE: If you have expressions separated by commas and want to parse them - /// individually instead of as a tuple, as done by [`Parser::parse_expression_list`] - /// use this function! + /// NOTE: If you have expressions separated by commas and want to parse them individually + /// instead of as a tuple, as done by [`Parser::parse_expression_list`] use this function. /// /// [Python grammar]: https://docs.python.org/3/reference/grammar.html - pub(super) fn parse_conditional_expression_or_higher( + pub(super) fn parse_conditional_expression_or_higher(&mut self) -> ParsedExpr { + self.parse_conditional_expression_or_higher_impl(ExpressionContext::default()) + } + + pub(super) fn parse_conditional_expression_or_higher_impl( &mut self, - allow_starred_expression: AllowStarredExpression, + context: ExpressionContext, ) -> ParsedExpr { if self.at(TokenKind::Lambda) { Expr::Lambda(self.parse_lambda_expr()).into() } else { let start = self.node_start(); - let parsed_expr = self.parse_simple_expression(allow_starred_expression); + let parsed_expr = self.parse_simple_expression(context); if self.at(TokenKind::If) { Expr::If(self.parse_if_expression(parsed_expr.expr, start)).into() @@ -266,28 +203,14 @@ impl<'src> Parser<'src> { /// This is a combination of the `disjunction`, `starred_expression`, `yield_expr` /// and `lambdef` rules of the [Python grammar]. /// - /// Note that this function parses yield and lambda expression but reports an error - /// as they're not allowed in this context. This is done for better error recovery. - /// Use [`Parser::parse_yield_expression_or_else`] to allow parsing yield expression. - /// Use [`Parser::parse_conditional_expression_or_higher`] or any methods which calls - /// into the specified method to allow parsing lambda expression. - /// - /// The caller can specify whether starred expression is allowed or not. This - /// doesn't affect the parsing of a starred expression as it will be parsed - /// nevertheless. But, if it is not allowed, an error is reported. + /// Note that this function parses lambda expression but reports an error as they're not + /// allowed in this context. This is done for better error recovery. + /// Use [`Parser::parse_conditional_expression_or_higher`] or any methods which calls into the + /// specified method to allow parsing lambda expression. /// /// [Python grammar]: https://docs.python.org/3/reference/grammar.html - fn parse_simple_expression( - &mut self, - allow_starred_expression: AllowStarredExpression, - ) -> ParsedExpr { - let parsed_expr = self.parse_expression_with_precedence(Precedence::Initial); - - if allow_starred_expression.is_no() && parsed_expr.is_unparenthesized_starred_expr() { - self.add_error(ParseErrorType::InvalidStarredExpressionUsage, &parsed_expr); - } - - parsed_expr + fn parse_simple_expression(&mut self, context: ExpressionContext) -> ParsedExpr { + self.parse_expression_with_precedence(Precedence::Initial, context) } /// Returns the binding power of the current token for a Pratt parser. @@ -345,10 +268,14 @@ impl<'src> Parser<'src> { /// This method uses the [Pratt parsing algorithm]. /// /// [Pratt parsing algorithm]: https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html - fn parse_expression_with_precedence(&mut self, previous_precedence: Precedence) -> ParsedExpr { + fn parse_expression_with_precedence( + &mut self, + previous_precedence: Precedence, + context: ExpressionContext, + ) -> ParsedExpr { let start = self.node_start(); - let lhs = self.parse_lhs_expression(previous_precedence); - self.parse_expression_with_precedence_recursive(lhs, previous_precedence, start) + let lhs = self.parse_lhs_expression(previous_precedence, context); + self.parse_expression_with_precedence_recursive(lhs, previous_precedence, context, start) } /// Parses an expression with binding power of at least `previous_precedence` given the @@ -357,6 +284,7 @@ impl<'src> Parser<'src> { &mut self, mut lhs: ParsedExpr, previous_precedence: Precedence, + context: ExpressionContext, start: TextSize, ) -> ParsedExpr { let mut progress = ParserProgress::default(); @@ -369,8 +297,7 @@ impl<'src> Parser<'src> { break; } - // Don't parse a `CompareExpr` if we are parsing a `Comprehension` or `ForStmt` - if matches!(token, TokenKind::In) && self.has_ctx(ParserCtxFlags::FOR_TARGET) { + if matches!(token, TokenKind::In) && context.is_in_excluded() { break; } @@ -381,29 +308,33 @@ impl<'src> Parser<'src> { self.bump(token); - // We need to create a dedicated node for boolean operations and - // comparison operations even though they are infix operators. + // We need to create a dedicated node for boolean operations and comparison operations + // even though they are infix operators. if token.is_bool_operator() { lhs = Expr::BoolOp(self.parse_bool_operation_expression( lhs.expr, start, token, operator_binding_power, - )) - .into(); - continue; - } else if token.is_compare_operator() { - lhs = Expr::Compare(self.parse_compare_expression( - lhs.expr, - start, - token, - operator_binding_power, + context, )) .into(); continue; } - let rhs = self.parse_expression_with_precedence(operator_binding_power); + if token.is_compare_operator() { + lhs = Expr::Compare(self.parse_compare_expression( + lhs.expr, + start, + token, + operator_binding_power, + context, + )) + .into(); + continue; + } + + let rhs = self.parse_expression_with_precedence(operator_binding_power, context); lhs.expr = Expr::BinOp(ast::ExprBinOp { left: Box::new(lhs.expr), @@ -425,12 +356,16 @@ impl<'src> Parser<'src> { /// is valid in that context. For example, a unary operator is not valid /// in an `await` expression in which case the `previous_precedence` would /// be [`Precedence::Await`]. - fn parse_lhs_expression(&mut self, previous_precedence: Precedence) -> ParsedExpr { + fn parse_lhs_expression( + &mut self, + previous_precedence: Precedence, + context: ExpressionContext, + ) -> ParsedExpr { let start = self.node_start(); let lhs = match self.current_token_kind() { unary_tok @ (TokenKind::Plus | TokenKind::Minus | TokenKind::Tilde) => { - let unary_expr = self.parse_unary_expression(); + let unary_expr = self.parse_unary_expression(context); if previous_precedence > Precedence::PosNegBitNot // > The power operator `**` binds less tightly than an arithmetic // > or bitwise unary operator on its right, that is, 2**-1 is 0.5. @@ -448,7 +383,7 @@ impl<'src> Parser<'src> { Expr::UnaryOp(unary_expr).into() } TokenKind::Not => { - let unary_expr = self.parse_unary_expression(); + let unary_expr = self.parse_unary_expression(context); if previous_precedence > Precedence::Not { self.add_error( ParseErrorType::OtherError( @@ -460,9 +395,10 @@ impl<'src> Parser<'src> { Expr::UnaryOp(unary_expr).into() } TokenKind::Star => { - let starred_expr = - self.parse_starred_expression(StarredExpressionPrecedence::Conditional); - if previous_precedence > Precedence::Initial { + let starred_expr = self.parse_starred_expression(context); + if previous_precedence > Precedence::Initial + || !context.is_starred_expression_allowed() + { self.add_error(ParseErrorType::InvalidStarredExpressionUsage, &starred_expr); } Expr::Starred(starred_expr).into() @@ -488,10 +424,12 @@ impl<'src> Parser<'src> { Expr::Lambda(lambda_expr).into() } TokenKind::Yield => { - // Yield expressions aren't allowed in this context but we'll still - // parse it and report an error for better recovery. let expr = self.parse_yield_expression(); - self.add_error(ParseErrorType::InvalidYieldExpressionUsage, &expr); + if previous_precedence > Precedence::Initial + || !context.is_yield_expression_allowed() + { + self.add_error(ParseErrorType::InvalidYieldExpressionUsage, &expr); + } expr.into() } _ => self.parse_atom(), @@ -511,7 +449,7 @@ impl<'src> Parser<'src> { /// /// [Python grammar]: https://docs.python.org/3/reference/grammar.html fn parse_expression_with_bitwise_or_precedence(&mut self) -> ParsedExpr { - let parsed_expr = self.parse_conditional_expression_or_higher(AllowStarredExpression::No); + let parsed_expr = self.parse_conditional_expression_or_higher(); if parsed_expr.is_parenthesized { // Parentheses resets the precedence, so we don't need to validate it. @@ -669,38 +607,11 @@ impl<'src> Parser<'src> { Expr::IpyEscapeCommand(self.parse_ipython_escape_command_expression()) } TokenKind::String | TokenKind::FStringStart => self.parse_strings(), - tok @ (TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace) => { - // We need to unset the `FOR_TARGET` in the context when parsing an expression - // inside a parentheses, curly brace or brackets otherwise the `in` operator of a - // comparison expression will not be parsed in a `for` target. - - // test_ok parenthesized_compare_expr_in_for - // for (x in y)[0] in iter: ... - // for (x in y).attr in iter: ... - - // test_err parenthesized_compare_expr_in_for - // for (x in y)() in iter: ... - // for (x in y) in iter: ... - // for (x in y, z) in iter: ... - // for [x in y, z] in iter: ... - // for {x in y, z} in iter: ... - let current_context = self.ctx - ParserCtxFlags::FOR_TARGET; - let saved_context = self.set_ctx(current_context); - - let expr = match tok { - TokenKind::Lpar => { - let parsed_expr = self.parse_parenthesized_expression(); - self.restore_ctx(current_context, saved_context); - return parsed_expr; - } - TokenKind::Lsqb => self.parse_list_like_expression(), - TokenKind::Lbrace => self.parse_set_or_dict_like_expression(), - _ => unreachable!(), - }; - - self.restore_ctx(current_context, saved_context); - expr + TokenKind::Lpar => { + return self.parse_parenthesized_expression(); } + TokenKind::Lsqb => self.parse_list_like_expression(), + TokenKind::Lbrace => self.parse_set_or_dict_like_expression(), kind => { if kind.is_keyword() { @@ -729,25 +640,14 @@ impl<'src> Parser<'src> { /// /// This method does nothing if the current token is not a candidate for a postfix expression. pub(super) fn parse_postfix_expression(&mut self, mut lhs: Expr, start: TextSize) -> Expr { - // test_ok for_in_target_postfix_expr - // for d[x in y] in target: ... - - // test_err for_in_target_postfix_expr - // for d(x in y) in target: ... - let current_context = self.ctx - ParserCtxFlags::FOR_TARGET; - let saved_context = self.set_ctx(current_context); - - lhs = loop { + loop { lhs = match self.current_token_kind() { TokenKind::Lpar => Expr::Call(self.parse_call_expression(lhs, start)), TokenKind::Lsqb => Expr::Subscript(self.parse_subscript_expression(lhs, start)), TokenKind::Dot => Expr::Attribute(self.parse_attribute_expression(lhs, start)), _ => break lhs, }; - }; - - self.restore_ctx(current_context, saved_context); - lhs + } } /// Parse a call expression. @@ -789,8 +689,7 @@ impl<'src> Parser<'src> { self.parse_comma_separated_list(RecoveryContextKind::Arguments, |parser| { let argument_start = parser.node_start(); if parser.eat(TokenKind::DoubleStar) { - let value = - parser.parse_conditional_expression_or_higher(AllowStarredExpression::No); + let value = parser.parse_conditional_expression_or_higher(); keywords.push(ast::Keyword { arg: None, @@ -801,8 +700,8 @@ impl<'src> Parser<'src> { seen_keyword_unpacking = true; } else { let start = parser.node_start(); - let mut parsed_expr = - parser.parse_named_expression_or_higher(AllowStarredExpression::Yes); + let mut parsed_expr = parser + .parse_named_expression_or_higher(ExpressionContext::starred_conditional()); match parser.current_token_kind() { TokenKind::Async | TokenKind::For => { @@ -850,8 +749,7 @@ impl<'src> Parser<'src> { } }; - let value = - parser.parse_conditional_expression_or_higher(AllowStarredExpression::No); + let value = parser.parse_conditional_expression_or_higher(); keywords.push(ast::Keyword { arg: Some(arg), @@ -980,7 +878,8 @@ impl<'src> Parser<'src> { let start = self.node_start(); let lower = if self.at_expr() { - let lower = self.parse_named_expression_or_higher(AllowStarredExpression::Yes); + let lower = + self.parse_named_expression_or_higher(ExpressionContext::starred_conditional()); if self.at_ts(NEWLINE_EOF_SET.union([TokenKind::Rsqb, TokenKind::Comma].into())) { return lower.expr; } @@ -1008,20 +907,14 @@ impl<'src> Parser<'src> { let upper = if self.at_ts(UPPER_END_SET) { None } else { - Some(Box::new( - self.parse_conditional_expression_or_higher(AllowStarredExpression::No) - .expr, - )) + Some(Box::new(self.parse_conditional_expression_or_higher().expr)) }; let step = if self.eat(TokenKind::Colon) { if self.at_ts(STEP_END_SET) { None } else { - Some(Box::new( - self.parse_conditional_expression_or_higher(AllowStarredExpression::No) - .expr, - )) + Some(Box::new(self.parse_conditional_expression_or_higher().expr)) } } else { None @@ -1045,7 +938,10 @@ impl<'src> Parser<'src> { /// If the parser isn't positioned at any of the unary operators. /// /// See: - pub(super) fn parse_unary_expression(&mut self) -> ast::ExprUnaryOp { + pub(super) fn parse_unary_expression( + &mut self, + context: ExpressionContext, + ) -> ast::ExprUnaryOp { let start = self.node_start(); let op = UnaryOp::try_from(self.current_token_kind()) @@ -1053,10 +949,10 @@ impl<'src> Parser<'src> { self.bump(self.current_token_kind()); let operand = if op.is_not() { - self.parse_expression_with_precedence(Precedence::Not) + self.parse_expression_with_precedence(Precedence::Not, context) } else { // plus, minus and tilde - self.parse_expression_with_precedence(Precedence::PosNegBitNot) + self.parse_expression_with_precedence(Precedence::PosNegBitNot, context) }; ast::ExprUnaryOp { @@ -1106,6 +1002,7 @@ impl<'src> Parser<'src> { start: TextSize, operator_token: TokenKind, operator_binding_power: Precedence, + context: ExpressionContext, ) -> ast::ExprBoolOp { let mut values = vec![lhs]; let mut progress = ParserProgress::default(); @@ -1115,7 +1012,8 @@ impl<'src> Parser<'src> { loop { progress.assert_progressing(self); - let parsed_expr = self.parse_expression_with_precedence(operator_binding_power); + let parsed_expr = + self.parse_expression_with_precedence(operator_binding_power, context); values.push(parsed_expr.expr); if !self.eat(operator_token) { @@ -1148,6 +1046,7 @@ impl<'src> Parser<'src> { start: TextSize, operator: TokenKind, operator_binding_power: Precedence, + context: ExpressionContext, ) -> ast::ExprCompare { let compare_operator = token_kind_to_cmp_op([operator, self.current_token_kind()]).unwrap(); @@ -1171,11 +1070,14 @@ impl<'src> Parser<'src> { loop { progress.assert_progressing(self); - let parsed_expr = self.parse_expression_with_precedence(operator_binding_power); + let parsed_expr = + self.parse_expression_with_precedence(operator_binding_power, context); comparators.push(parsed_expr.expr); let next_operator = self.current_token_kind(); - if !next_operator.is_compare_operator() { + if !next_operator.is_compare_operator() + || (matches!(next_operator, TokenKind::In) && context.is_in_excluded()) + { break; } self.bump(next_operator); // compare operator @@ -1525,7 +1427,7 @@ impl<'src> Parser<'src> { // f"{*}" // f"{*x and y}" // f"{*yield x}" - let value = self.parse_yield_expression_or_else(Parser::parse_star_expression_list); + let value = self.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or()); if !value.is_parenthesized && value.expr.is_lambda_expr() { // TODO(dhruvmanila): This requires making some changes in lambda expression @@ -1655,7 +1557,8 @@ impl<'src> Parser<'src> { } // Parse the first element with a more general rule and limit it later. - let first_element = self.parse_star_expression_or_higher(AllowNamedExpression::Yes); + let first_element = + self.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or()); match self.current_token_kind() { TokenKind::Async | TokenKind::For => { @@ -1716,7 +1619,8 @@ impl<'src> Parser<'src> { // For dictionary expressions, the key uses the `expression` rule while for // set expressions, the element uses the `star_expression` rule. So, use the // one that is more general and limit it later. - let key_or_element = self.parse_star_expression_or_higher(AllowNamedExpression::Yes); + let key_or_element = + self.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or()); match self.current_token_kind() { TokenKind::Async | TokenKind::For => { @@ -1747,7 +1651,7 @@ impl<'src> Parser<'src> { } self.bump(TokenKind::Colon); - let value = self.parse_conditional_expression_or_higher(AllowStarredExpression::No); + let value = self.parse_conditional_expression_or_higher(); if matches!(self.current_token_kind(), TokenKind::Async | TokenKind::For) { Expr::DictComp(self.parse_dictionary_comprehension_expression( @@ -1798,19 +1702,16 @@ impl<'src> Parser<'src> { // Use the more general rule of the three to parse the first element // and limit it later. - let mut parsed_expr = self.parse_yield_expression_or_else(|p| { - p.parse_star_expression_or_higher(AllowNamedExpression::Yes) - }); + let mut parsed_expr = + self.parse_named_expression_or_higher(ExpressionContext::yield_or_starred_bitwise_or()); match self.current_token_kind() { TokenKind::Comma => { // grammar: `tuple` - let tuple = self.parse_tuple_expression( - parsed_expr.expr, - start, - Parenthesized::Yes, - |parser| parser.parse_star_expression_or_higher(AllowNamedExpression::Yes), - ); + let tuple = + self.parse_tuple_expression(parsed_expr.expr, start, Parenthesized::Yes, |p| { + p.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or()) + }); ParsedExpr { expr: tuple.into(), @@ -1898,7 +1799,7 @@ impl<'src> Parser<'src> { self.parse_comma_separated_list(RecoveryContextKind::ListElements, |parser| { elts.push( parser - .parse_star_expression_or_higher(AllowNamedExpression::Yes) + .parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or()) .expr, ); }); @@ -1925,7 +1826,7 @@ impl<'src> Parser<'src> { self.parse_comma_separated_list(RecoveryContextKind::SetElements, |parser| { elts.push( parser - .parse_star_expression_or_higher(AllowNamedExpression::Yes) + .parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or()) .expr, ); }); @@ -1962,18 +1863,10 @@ impl<'src> Parser<'src> { // which requires limiting the expression. values.push(parser.parse_expression_with_bitwise_or_precedence().expr); } else { - keys.push(Some( - parser - .parse_conditional_expression_or_higher(AllowStarredExpression::No) - .expr, - )); + keys.push(Some(parser.parse_conditional_expression_or_higher().expr)); parser.expect(TokenKind::Colon); - values.push( - parser - .parse_conditional_expression_or_higher(AllowStarredExpression::No) - .expr, - ); + values.push(parser.parse_conditional_expression_or_higher().expr); } }); @@ -2027,15 +1920,14 @@ impl<'src> Parser<'src> { self.bump(TokenKind::For); }; - let saved_context = self.set_ctx(ParserCtxFlags::FOR_TARGET); - let mut target = self.parse_expression_list(AllowStarredExpression::Yes); - self.restore_ctx(ParserCtxFlags::FOR_TARGET, saved_context); + let mut target = + self.parse_expression_list(ExpressionContext::starred_conditional().with_in_excluded()); helpers::set_expr_ctx(&mut target.expr, ExprContext::Store); self.validate_assignment_target(&target.expr); self.expect(TokenKind::In); - let iter = self.parse_simple_expression(AllowStarredExpression::No); + let iter = self.parse_simple_expression(ExpressionContext::default()); let mut ifs = vec![]; let mut progress = ParserProgress::default(); @@ -2043,7 +1935,7 @@ impl<'src> Parser<'src> { while self.eat(TokenKind::If) { progress.assert_progressing(self); - let parsed_expr = self.parse_simple_expression(AllowStarredExpression::No); + let parsed_expr = self.parse_simple_expression(ExpressionContext::default()); ifs.push(parsed_expr.expr); } @@ -2177,18 +2069,15 @@ impl<'src> Parser<'src> { /// If the parser isn't positioned at a `*` token. /// /// [Python grammar]: https://docs.python.org/3/reference/grammar.html - fn parse_starred_expression( - &mut self, - precedence: StarredExpressionPrecedence, - ) -> ast::ExprStarred { + fn parse_starred_expression(&mut self, context: ExpressionContext) -> ast::ExprStarred { let start = self.node_start(); self.bump(TokenKind::Star); - let parsed_expr = match precedence { + let parsed_expr = match context.starred_expression_precedence() { StarredExpressionPrecedence::Conditional => { - self.parse_conditional_expression_or_higher(AllowStarredExpression::No) + self.parse_conditional_expression_or_higher_impl(context) } - StarredExpressionPrecedence::BitOr => { + StarredExpressionPrecedence::BitwiseOr => { self.parse_expression_with_bitwise_or_precedence() } }; @@ -2211,7 +2100,8 @@ impl<'src> Parser<'src> { let start = self.node_start(); self.bump(TokenKind::Await); - let parsed_expr = self.parse_expression_with_precedence(Precedence::Await); + let parsed_expr = + self.parse_expression_with_precedence(Precedence::Await, ExpressionContext::default()); ast::ExprAwait { value: Box::new(parsed_expr.expr), @@ -2219,23 +2109,6 @@ impl<'src> Parser<'src> { } } - /// Parses a yield expression if the parser is positioned at a `yield` token - /// or calls the given closure to parse an expression. - /// - /// This method is used where the grammar allows a `yield` expression or an - /// alternative expression. For example, the grammar for a parenthesized - /// expression is `(yield_expr | named_expression)`. - pub(super) fn parse_yield_expression_or_else(&mut self, f: F) -> ParsedExpr - where - F: Fn(&mut Parser<'src>) -> ParsedExpr, - { - if self.at(TokenKind::Yield) { - self.parse_yield_expression().into() - } else { - f(self) - } - } - /// Parses a `yield` expression. /// /// # Panics @@ -2251,9 +2124,12 @@ impl<'src> Parser<'src> { return self.parse_yield_from_expression(start); } - let value = self - .at_expr() - .then(|| Box::new(self.parse_star_expression_list().expr)); + let value = self.at_expr().then(|| { + Box::new( + self.parse_expression_list(ExpressionContext::starred_bitwise_or()) + .expr, + ) + }); Expr::Yield(ast::ExprYield { value, @@ -2283,7 +2159,9 @@ impl<'src> Parser<'src> { // If we didn't use the `parse_expression_list` method here, the parser // would have stopped at the comma. Then, the outer expression would // have been a tuple expression with two elements: `yield from x` and `y`. - let expr = self.parse_expression_list(AllowStarredExpression::No).expr; + let expr = self + .parse_expression_list(ExpressionContext::default()) + .expr; match &expr { Expr::Tuple(tuple) if !tuple.parenthesized => { @@ -2317,7 +2195,7 @@ impl<'src> Parser<'src> { } helpers::set_expr_ctx(&mut target, ExprContext::Store); - let value = self.parse_conditional_expression_or_higher(AllowStarredExpression::No); + let value = self.parse_conditional_expression_or_higher(); ast::ExprNamed { target: Box::new(target), @@ -2364,7 +2242,7 @@ impl<'src> Parser<'src> { // test_err lambda_body_with_yield_expr // lambda x: yield y // lambda x: yield from y - let body = self.parse_conditional_expression_or_higher(AllowStarredExpression::No); + let body = self.parse_conditional_expression_or_higher(); ast::ExprLambda { body: Box::new(body.expr), @@ -2383,11 +2261,11 @@ impl<'src> Parser<'src> { pub(super) fn parse_if_expression(&mut self, body: Expr, start: TextSize) -> ast::ExprIf { self.bump(TokenKind::If); - let test = self.parse_simple_expression(AllowStarredExpression::No); + let test = self.parse_simple_expression(ExpressionContext::default()); self.expect(TokenKind::Else); - let orelse = self.parse_conditional_expression_or_higher(AllowStarredExpression::No); + let orelse = self.parse_conditional_expression_or_higher(); ast::ExprIf { body: Box::new(body), @@ -2589,32 +2467,116 @@ pub(super) enum GeneratorExpressionInParentheses { }, } +/// Represents the precedence used for parsing the value part of a starred expression. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum StarredExpressionPrecedence { - BitOr, +pub(super) enum StarredExpressionPrecedence { + /// Matches `'*' bitwise_or` which is part of the `star_expression` rule in the + /// [Python grammar](https://docs.python.org/3/reference/grammar.html). + BitwiseOr, + + /// Matches `'*' expression` which is part of the `starred_expression` rule in the + /// [Python grammar](https://docs.python.org/3/reference/grammar.html). Conditional, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum AllowNamedExpression { - Yes, - No, -} +/// Represents the expression parsing context. +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] +pub(super) struct ExpressionContext(ExpressionContextFlags); -impl AllowNamedExpression { - const fn is_yes(self) -> bool { - matches!(self, AllowNamedExpression::Yes) +bitflags! { + #[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] + struct ExpressionContextFlags: u8 { + /// This flag is set when the `in` keyword should be excluded from a comparison expression. + /// It is to avoid ambiguity in `for ... in ...` statements. + const EXCLUDE_IN = 1 << 0; + + /// This flag is set when a starred expression should be allowed. This doesn't affect the + /// parsing of a starred expression as it will be parsed nevertheless. But, if it is not + /// allowed, an error is reported. + const ALLOW_STARRED_EXPRESSION = 1 << 1; + + /// This flag is set when the value of a starred expression should be limited to bitwise OR + /// precedence. Matches the `* bitwise_or` grammar rule if set. + const STARRED_BITWISE_OR_PRECEDENCE = 1 << 2; + + /// This flag is set when a yield expression should be allowed. This doesn't affect the + /// parsing of a yield expression as it will be parsed nevertheless. But, if it is not + /// allowed, an error is reported. + const ALLOW_YIELD_EXPRESSION = 1 << 3; } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum AllowStarredExpression { - Yes, - No, -} +impl ExpressionContext { + /// Create a new context allowing starred expression at conditional precedence. + pub(super) fn starred_conditional() -> Self { + ExpressionContext::default() + .with_starred_expression_allowed(StarredExpressionPrecedence::Conditional) + } -impl AllowStarredExpression { - const fn is_no(self) -> bool { - matches!(self, AllowStarredExpression::No) + /// Create a new context allowing starred expression at bitwise OR precedence. + pub(super) fn starred_bitwise_or() -> Self { + ExpressionContext::default() + .with_starred_expression_allowed(StarredExpressionPrecedence::BitwiseOr) + } + + /// Create a new context allowing starred expression at bitwise OR precedence or yield + /// expression. + pub(super) fn yield_or_starred_bitwise_or() -> Self { + ExpressionContext::starred_bitwise_or().with_yield_expression_allowed() + } + + /// Returns a new [`ExpressionContext`] which allows starred expression with the given + /// precedence. + fn with_starred_expression_allowed(self, precedence: StarredExpressionPrecedence) -> Self { + let mut flags = self.0 | ExpressionContextFlags::ALLOW_STARRED_EXPRESSION; + match precedence { + StarredExpressionPrecedence::BitwiseOr => { + flags |= ExpressionContextFlags::STARRED_BITWISE_OR_PRECEDENCE; + } + StarredExpressionPrecedence::Conditional => { + flags -= ExpressionContextFlags::STARRED_BITWISE_OR_PRECEDENCE; + } + } + ExpressionContext(flags) + } + + /// Returns a new [`ExpressionContext`] which allows yield expression. + fn with_yield_expression_allowed(self) -> Self { + ExpressionContext(self.0 | ExpressionContextFlags::ALLOW_YIELD_EXPRESSION) + } + + /// Returns a new [`ExpressionContext`] which excludes `in` as part of a comparison expression. + pub(super) fn with_in_excluded(self) -> Self { + ExpressionContext(self.0 | ExpressionContextFlags::EXCLUDE_IN) + } + + /// Returns `true` if the `in` keyword should be excluded from a comparison expression. + const fn is_in_excluded(self) -> bool { + self.0.contains(ExpressionContextFlags::EXCLUDE_IN) + } + + /// Returns `true` if starred expressions are allowed. + const fn is_starred_expression_allowed(self) -> bool { + self.0 + .contains(ExpressionContextFlags::ALLOW_STARRED_EXPRESSION) + } + + /// Returns `true` if yield expressions are allowed. + const fn is_yield_expression_allowed(self) -> bool { + self.0 + .contains(ExpressionContextFlags::ALLOW_YIELD_EXPRESSION) + } + + /// Returns the [`StarredExpressionPrecedence`] for the context, regardless of whether starred + /// expressions are allowed or not. + const fn starred_expression_precedence(self) -> StarredExpressionPrecedence { + if self + .0 + .contains(ExpressionContextFlags::STARRED_BITWISE_OR_PRECEDENCE) + { + StarredExpressionPrecedence::BitwiseOr + } else { + StarredExpressionPrecedence::Conditional + } } } diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index d1c9b7e2a4..1f1db32b9b 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -1,7 +1,6 @@ use std::cmp::Ordering; use bitflags::bitflags; -use drop_bomb::DebugDropBomb; use ast::Mod; use ruff_python_ast as ast; @@ -16,7 +15,7 @@ use crate::{ Mode, ParseError, ParseErrorType, Tok, TokenKind, }; -use self::expression::AllowStarredExpression; +use self::expression::ExpressionContext; mod expression; mod helpers; @@ -77,13 +76,6 @@ pub(crate) struct Parser<'src> { /// Stores all the syntax errors found during the parsing. errors: Vec, - /// This tracks the current expression or statement being parsed. - /// - /// The `ctx` is also used to create custom error messages and forbid certain - /// expressions or statements of being parsed. The `ctx` should be empty after - /// an expression or statement is done parsing. - ctx: ParserCtxFlags, - /// Specify the mode in which the code will be parsed. mode: Mode, @@ -123,7 +115,6 @@ impl<'src> Parser<'src> { mode, source, errors: Vec::new(), - ctx: ParserCtxFlags::empty(), tokens, recovery_context: RecoveryContext::empty(), last_token_end: tokens_range.start(), @@ -136,7 +127,7 @@ impl<'src> Parser<'src> { pub(crate) fn parse_program(mut self) -> Program { let ast = if self.mode == Mode::Expression { let start = self.node_start(); - let parsed_expr = self.parse_expression_list(AllowStarredExpression::No); + let parsed_expr = self.parse_expression_list(ExpressionContext::default()); // All of the remaining newlines are actually going to be non-logical newlines. self.eat(TokenKind::Newline); @@ -185,9 +176,6 @@ impl<'src> Parser<'src> { } fn finish(self) -> Vec { - // After parsing, the `ctx` and `ctx_stack` should be empty. - // If it's not, you probably forgot to call `clear_ctx` somewhere. - assert_eq!(self.ctx, ParserCtxFlags::empty()); assert_eq!( self.current_token_kind(), TokenKind::EndOfFile, @@ -232,29 +220,6 @@ impl<'src> Parser<'src> { merged } - #[inline] - #[must_use] - fn set_ctx(&mut self, ctx: ParserCtxFlags) -> SavedParserContext { - SavedParserContext { - flags: std::mem::replace(&mut self.ctx, ctx), - bomb: DebugDropBomb::new( - "You must restore the old parser context explicit by calling `restore_ctx`", - ), - } - } - - #[inline] - fn restore_ctx(&mut self, current: ParserCtxFlags, mut saved_context: SavedParserContext) { - assert_eq!(self.ctx, current); - saved_context.bomb.defuse(); - self.ctx = saved_context.flags; - } - - #[inline] - fn has_ctx(&self, ctx: ParserCtxFlags) -> bool { - self.ctx.intersects(ctx) - } - /// Returns the start position for a node that starts at the current token. fn node_start(&self) -> TextSize { self.current_token_range().start() @@ -675,13 +640,6 @@ impl SequenceMatchPatternParentheses { } } -bitflags! { - #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] - struct ParserCtxFlags: u8 { - const FOR_TARGET = 1 << 2; - } -} - #[derive(Debug, PartialEq, Copy, Clone)] enum FunctionKind { /// A lambda expression, e.g., `lambda x: x` @@ -1327,9 +1285,3 @@ impl RecoveryContext { }) } } - -#[derive(Debug)] -struct SavedParserContext { - flags: ParserCtxFlags, - bomb: DebugDropBomb, -} diff --git a/crates/ruff_python_parser/src/parser/pattern.rs b/crates/ruff_python_parser/src/parser/pattern.rs index 9395676227..743722119d 100644 --- a/crates/ruff_python_parser/src/parser/pattern.rs +++ b/crates/ruff_python_parser/src/parser/pattern.rs @@ -6,6 +6,8 @@ use crate::parser::{recovery, Parser, RecoveryContextKind, SequenceMatchPatternP use crate::token_set::TokenSet; use crate::{ParseErrorType, Tok, TokenKind}; +use super::expression::ExpressionContext; + /// The set of tokens that can start a literal pattern. const LITERAL_PATTERN_START_SET: TokenSet = TokenSet::new([ TokenKind::None, @@ -483,7 +485,7 @@ impl<'src> Parser<'src> { TokenKind::Int | TokenKind::Float | TokenKind::Complex ) => { - let unary_expr = self.parse_unary_expression(); + let unary_expr = self.parse_unary_expression(ExpressionContext::default()); if unary_expr.op.is_u_add() { self.add_error( diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs index 09f779be04..8b534ba024 100644 --- a/crates/ruff_python_parser/src/parser/statement.rs +++ b/crates/ruff_python_parser/src/parser/statement.rs @@ -11,13 +11,12 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::parser::expression::{GeneratorExpressionInParentheses, ParsedExpr, EXPR_SET}; use crate::parser::progress::ParserProgress; use crate::parser::{ - helpers, FunctionKind, Parser, ParserCtxFlags, RecoveryContext, RecoveryContextKind, - WithItemKind, + helpers, FunctionKind, Parser, RecoveryContext, RecoveryContextKind, WithItemKind, }; use crate::token_set::TokenSet; use crate::{Mode, ParseErrorType, Tok, TokenKind}; -use super::expression::{AllowNamedExpression, AllowStarredExpression, Precedence}; +use super::expression::{ExpressionContext, Precedence}; use super::Parenthesized; /// Tokens that represent compound statements. @@ -262,7 +261,7 @@ impl<'src> Parser<'src> { // simple_stmt: `... | yield_stmt | star_expressions | ...` let parsed_expr = - self.parse_yield_expression_or_else(Parser::parse_star_expression_list); + self.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or()); if self.at(TokenKind::Equal) { Stmt::Assign(self.parse_assign_statement(parsed_expr, start)) @@ -309,8 +308,9 @@ impl<'src> Parser<'src> { |parser| { // Allow starred expression to raise a better error message for // an invalid delete target later. - let mut target = - parser.parse_conditional_expression_or_higher(AllowStarredExpression::Yes); + let mut target = parser.parse_conditional_expression_or_higher_impl( + ExpressionContext::starred_conditional(), + ); helpers::set_expr_ctx(&mut target.expr, ExprContext::Del); // test_err invalid_del_target @@ -356,9 +356,12 @@ impl<'src> Parser<'src> { // return yield from x // return x := 1 // return *x and y - let value = self - .at_expr() - .then(|| Box::new(self.parse_star_expression_list().expr)); + let value = self.at_expr().then(|| { + Box::new( + self.parse_expression_list(ExpressionContext::starred_bitwise_or()) + .expr, + ) + }); ast::StmtReturn { range: self.node_range(start), @@ -384,7 +387,7 @@ impl<'src> Parser<'src> { // raise *x // raise yield x // raise x := 1 - let exc = self.parse_expression_list(AllowStarredExpression::No); + let exc = self.parse_expression_list(ExpressionContext::default()); if let Some(ast::ExprTuple { parenthesized: false, @@ -406,7 +409,7 @@ impl<'src> Parser<'src> { // raise x from *y // raise x from yield y // raise x from y := 1 - let cause = self.parse_expression_list(AllowStarredExpression::No); + let cause = self.parse_expression_list(ExpressionContext::default()); if let Some(ast::ExprTuple { parenthesized: false, @@ -714,7 +717,7 @@ impl<'src> Parser<'src> { // assert assert x // assert yield x // assert x := 1 - let test = self.parse_conditional_expression_or_higher(AllowStarredExpression::No); + let test = self.parse_conditional_expression_or_higher(); let msg = if self.eat(TokenKind::Comma) { if self.at_expr() { @@ -723,10 +726,7 @@ impl<'src> Parser<'src> { // assert False, assert x // assert False, yield x // assert False, x := 1 - Some(Box::new( - self.parse_conditional_expression_or_higher(AllowStarredExpression::No) - .expr, - )) + Some(Box::new(self.parse_conditional_expression_or_higher().expr)) } else { // test_err assert_empty_msg // assert x, @@ -854,7 +854,7 @@ impl<'src> Parser<'src> { // type x = yield y // type x = yield from y // type x = x := 1 - let value = self.parse_conditional_expression_or_higher(AllowStarredExpression::No); + let value = self.parse_conditional_expression_or_higher(); ast::StmtTypeAlias { name: Box::new(name), @@ -1014,7 +1014,8 @@ impl<'src> Parser<'src> { // x = *lambda x: x // x = x := 1 - let mut value = self.parse_yield_expression_or_else(Parser::parse_star_expression_list); + let mut value = + self.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or()); if self.at(TokenKind::Equal) { // This path is only taken when there are more than one assignment targets. @@ -1022,7 +1023,7 @@ impl<'src> Parser<'src> { parser.bump(TokenKind::Equal); let mut parsed_expr = - parser.parse_yield_expression_or_else(Parser::parse_star_expression_list); + parser.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or()); std::mem::swap(&mut value, &mut parsed_expr); @@ -1092,7 +1093,7 @@ impl<'src> Parser<'src> { // test_err ann_assign_stmt_type_alias_annotation // a: type X = int // lambda: type X = int - let annotation = self.parse_conditional_expression_or_higher(AllowStarredExpression::No); + let annotation = self.parse_conditional_expression_or_higher(); let value = if self.eat(TokenKind::Equal) { if self.at_expr() { @@ -1101,7 +1102,7 @@ impl<'src> Parser<'src> { // x: Any = x := 1 // x: list = [x, *a | b, *a or b] Some(Box::new( - self.parse_yield_expression_or_else(Parser::parse_star_expression_list) + self.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or()) .expr, )) } else { @@ -1170,7 +1171,7 @@ impl<'src> Parser<'src> { // x += *yield from x // x += *lambda x: x // x += y := 1 - let value = self.parse_yield_expression_or_else(Parser::parse_star_expression_list); + let value = self.parse_expression_list(ExpressionContext::yield_or_starred_bitwise_or()); ast::StmtAugAssign { target: Box::new(target.expr), @@ -1198,7 +1199,7 @@ impl<'src> Parser<'src> { // test_err if_stmt_missing_test // if : ... - let test = self.parse_named_expression_or_higher(AllowStarredExpression::No); + let test = self.parse_named_expression_or_higher(ExpressionContext::default()); // test_err if_stmt_missing_colon // if x @@ -1253,7 +1254,7 @@ impl<'src> Parser<'src> { // elif yield x: // pass Some( - self.parse_named_expression_or_higher(AllowStarredExpression::No) + self.parse_named_expression_or_higher(ExpressionContext::default()) .expr, ) } else { @@ -1414,7 +1415,7 @@ impl<'src> Parser<'src> { // pass // except* *x: // pass - let parsed_expr = self.parse_expression_list(AllowStarredExpression::No); + let parsed_expr = self.parse_expression_list(ExpressionContext::default()); if matches!( parsed_expr.expr, Expr::Tuple(ast::ExprTuple { @@ -1522,22 +1523,31 @@ impl<'src> Parser<'src> { fn parse_for_statement(&mut self, start: TextSize) -> ast::StmtFor { self.bump(TokenKind::For); - // This is to avoid the ambiguity of the `in` token which is used in - // both the `for` statement and the comparison expression. For example: - // - // ```python - // for x in y: - // # ^^^^^^ - // # This is not a comparison expression - // pass - // ``` - let saved_context = self.set_ctx(ParserCtxFlags::FOR_TARGET); - // test_err for_stmt_missing_target // for in x: ... - let mut target = self.parse_expression_list(AllowStarredExpression::Yes); - self.restore_ctx(ParserCtxFlags::FOR_TARGET, saved_context); + // test_ok for_in_target_valid_expr + // for d[x in y] in target: ... + // for (x in y)[0] in iter: ... + // for (x in y).attr in iter: ... + + // test_err for_stmt_invalid_target_in_keyword + // for d(x in y) in target: ... + // for (x in y)() in iter: ... + // for (x in y) in iter: ... + // for (x in y, z) in iter: ... + // for [x in y, z] in iter: ... + // for {x in y, z} in iter: ... + + // test_err for_stmt_invalid_target_binary_expr + // for x not in y in z: ... + // for x == y in z: ... + // for x or y in z: ... + // for -x in y: ... + // for not x in y: ... + // for x | y in z: ... + let mut target = + self.parse_expression_list(ExpressionContext::starred_conditional().with_in_excluded()); helpers::set_expr_ctx(&mut target.expr, ExprContext::Store); @@ -1547,6 +1557,7 @@ impl<'src> Parser<'src> { // for *x and y in z: ... // for *x | y in z: ... // for await x in z: ... + // for yield x in y: ... // for [x, 1, y, *["a"]] in z: ... self.validate_assignment_target(&target.expr); @@ -1563,7 +1574,7 @@ impl<'src> Parser<'src> { // for x in *a and b: ... // for x in yield a: ... // for target in x := 1: ... - let iter = self.parse_star_expression_list(); + let iter = self.parse_expression_list(ExpressionContext::starred_bitwise_or()); self.expect(TokenKind::Colon); @@ -1607,7 +1618,7 @@ impl<'src> Parser<'src> { // while yield x: ... // while a, b: ... // while a := 1, b: ... - let test = self.parse_named_expression_or_higher(AllowStarredExpression::No); + let test = self.parse_named_expression_or_higher(ExpressionContext::default()); // test_err while_stmt_missing_colon // while ( @@ -1689,7 +1700,7 @@ impl<'src> Parser<'src> { // def foo() -> *int: ... // def foo() -> (*int): ... // def foo() -> yield x: ... - let returns = self.parse_expression_list(AllowStarredExpression::No); + let returns = self.parse_expression_list(ExpressionContext::default()); if matches!( returns.expr, @@ -2165,6 +2176,7 @@ impl<'src> Parser<'src> { self.parse_expression_with_precedence_recursive( lhs.into(), Precedence::Initial, + ExpressionContext::default(), start, ) .expr @@ -2215,9 +2227,8 @@ impl<'src> Parser<'src> { // // Thus, we can conclude that the grammar used should be: // (yield_expr | star_named_expression) - let parsed_expr = self.parse_yield_expression_or_else(|p| { - p.parse_star_expression_or_higher(AllowNamedExpression::Yes) - }); + let parsed_expr = self + .parse_named_expression_or_higher(ExpressionContext::yield_or_starred_bitwise_or()); if matches!(self.current_token_kind(), TokenKind::Async | TokenKind::For) { if parsed_expr.is_unparenthesized_starred_expr() { @@ -2279,7 +2290,7 @@ impl<'src> Parser<'src> { } else { // If it's not in an ambiguous state, then the grammar of the with item // should be used which is `expression`. - self.parse_conditional_expression_or_higher(AllowStarredExpression::No) + self.parse_conditional_expression_or_higher() }; let optional_vars = self @@ -2305,7 +2316,8 @@ impl<'src> Parser<'src> { fn parse_with_item_optional_vars(&mut self) -> ParsedExpr { self.bump(TokenKind::As); - let mut target = self.parse_conditional_expression_or_higher(AllowStarredExpression::Yes); + let mut target = self + .parse_conditional_expression_or_higher_impl(ExpressionContext::starred_conditional()); // This has the same semantics as an assignment target. self.validate_assignment_target(&target.expr); @@ -2336,7 +2348,8 @@ impl<'src> Parser<'src> { // // First try with `star_named_expression`, then if there's no comma, // we'll restrict it to `named_expression`. - let subject = self.parse_star_expression_or_higher(AllowNamedExpression::Yes); + let subject = + self.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or()); // test_ok match_stmt_subject_expr // match x := 1: @@ -2360,7 +2373,7 @@ impl<'src> Parser<'src> { let subject = if self.at(TokenKind::Comma) { let tuple = self.parse_tuple_expression(subject.expr, subject_start, Parenthesized::No, |p| { - p.parse_star_expression_or_higher(AllowNamedExpression::Yes) + p.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or()) }); Expr::Tuple(tuple).into() @@ -2470,7 +2483,7 @@ impl<'src> Parser<'src> { // match x: // case y if yield x: ... Some(Box::new( - self.parse_named_expression_or_higher(AllowStarredExpression::No) + self.parse_named_expression_or_higher(ExpressionContext::default()) .expr, )) } else { @@ -2588,7 +2601,7 @@ impl<'src> Parser<'src> { // @yield x // @yield from x // def foo(): ... - let parsed_expr = self.parse_named_expression_or_higher(AllowStarredExpression::No); + let parsed_expr = self.parse_named_expression_or_higher(ExpressionContext::default()); decorators.push(ast::Decorator { expression: parsed_expr.expr, @@ -2744,7 +2757,9 @@ impl<'src> Parser<'src> { // def foo(*args: *int or str): ... // def foo(*args: *yield x): ... // # def foo(*args: **int): ... - self.parse_star_expression_or_higher(AllowNamedExpression::No) + self.parse_conditional_expression_or_higher_impl( + ExpressionContext::starred_bitwise_or(), + ) } AllowStarAnnotation::No => { // test_ok param_with_annotation @@ -2757,7 +2772,7 @@ impl<'src> Parser<'src> { // def foo(arg: *int): ... // def foo(arg: yield int): ... // def foo(arg: x := int): ... - self.parse_conditional_expression_or_higher(AllowStarredExpression::No) + self.parse_conditional_expression_or_higher() } }; Some(Box::new(parsed_expr.expr)) @@ -2809,10 +2824,7 @@ impl<'src> Parser<'src> { // def foo(x=*int): ... // def foo(x=(*int)): ... // def foo(x=yield y): ... - Some(Box::new( - self.parse_conditional_expression_or_higher(AllowStarredExpression::No) - .expr, - )) + Some(Box::new(self.parse_conditional_expression_or_higher().expr)) } else { // test_err param_missing_default // def foo(x=): ... @@ -3176,10 +3188,7 @@ impl<'src> Parser<'src> { // type X[T: yield x] = int // type X[T: yield from x] = int // type X[T: x := int] = int - Some(Box::new( - self.parse_conditional_expression_or_higher(AllowStarredExpression::No) - .expr, - )) + Some(Box::new(self.parse_conditional_expression_or_higher().expr)) } else { // test_err type_param_missing_bound // type X[T: ] = int diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap index 1c663c7564..5d9c56c013 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__star_expression_precedence.py.snap @@ -346,24 +346,29 @@ Module( ExprList { range: 187..199, elts: [ - Starred( - ExprStarred { - range: 188..190, - value: Name( - ExprName { - range: 189..190, - id: "x", - ctx: Load, + Named( + ExprNamed { + range: 188..195, + target: Starred( + ExprStarred { + range: 188..190, + value: Name( + ExprName { + range: 189..190, + id: "x", + ctx: Store, + }, + ), + ctx: Store, }, ), - ctx: Load, - }, - ), - NumberLiteral( - ExprNumberLiteral { - range: 194..195, - value: Int( - 2, + value: NumberLiteral( + ExprNumberLiteral { + range: 194..195, + value: Int( + 2, + ), + }, ), }, ), @@ -458,5 +463,5 @@ Module( 8 | [*x if True else y, z] 9 | [*lambda x: x, z] 10 | [*x := 2, z] - | ^^ Syntax Error: Expected ',', found ':=' + | ^^ Syntax Error: Assignment expression target must be an identifier | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__named__invalid_target.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__named__invalid_target.py.snap index fbdfb36360..ca8a03f58f 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__named__invalid_target.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__named__invalid_target.py.snap @@ -84,30 +84,30 @@ Module( ), Expr( StmtExpr { - range: 81..84, - value: Starred( - ExprStarred { - range: 82..84, - value: Name( - ExprName { - range: 83..84, - id: "x", - ctx: Load, + range: 81..90, + value: Named( + ExprNamed { + range: 82..89, + target: Starred( + ExprStarred { + range: 82..84, + value: Name( + ExprName { + range: 83..84, + id: "x", + ctx: Store, + }, + ), + ctx: Store, }, ), - ctx: Load, - }, - ), - }, - ), - Expr( - StmtExpr { - range: 88..89, - value: NumberLiteral( - ExprNumberLiteral { - range: 88..89, - value: Int( - 1, + value: NumberLiteral( + ExprNumberLiteral { + range: 88..89, + value: Int( + 1, + ), + }, ), }, ), @@ -198,34 +198,7 @@ Module( 3 | (x.y := 1) 4 | (x[y] := 1) 5 | (*x := 1) - | ^^ Syntax Error: Starred expression cannot be used here -6 | ([x, y] := [1, 2]) - | - - - | -3 | (x.y := 1) -4 | (x[y] := 1) -5 | (*x := 1) - | ^^ Syntax Error: Expected ')', found ':=' -6 | ([x, y] := [1, 2]) - | - - - | -3 | (x.y := 1) -4 | (x[y] := 1) -5 | (*x := 1) - | ^ Syntax Error: Expected a statement -6 | ([x, y] := [1, 2]) - | - - - | -3 | (x.y := 1) -4 | (x[y] := 1) -5 | (*x := 1) - | ^ Syntax Error: Expected a statement + | ^^ Syntax Error: Assignment expression target must be an identifier 6 | ([x, y] := [1, 2]) | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap index 21afcfe655..49b3005abf 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__tuple_starred_expr.py.snap @@ -491,34 +491,34 @@ Module( ), Expr( StmtExpr { - range: 323..326, - value: Starred( - ExprStarred { - range: 324..326, - value: Name( - ExprName { - range: 325..326, - id: "x", - ctx: Load, - }, - ), - ctx: Load, - }, - ), - }, - ), - Expr( - StmtExpr { - range: 330..343, + range: 323..344, value: Tuple( ExprTuple { - range: 330..343, + range: 323..344, elts: [ - NumberLiteral( - ExprNumberLiteral { - range: 330..331, - value: Int( - 2, + Named( + ExprNamed { + range: 324..331, + target: Starred( + ExprStarred { + range: 324..326, + value: Name( + ExprName { + range: 325..326, + id: "x", + ctx: Store, + }, + ), + ctx: Store, + }, + ), + value: NumberLiteral( + ExprNumberLiteral { + range: 330..331, + value: Int( + 2, + ), + }, ), }, ), @@ -529,30 +529,35 @@ Module( ctx: Load, }, ), - Starred( - ExprStarred { - range: 336..338, - value: Name( - ExprName { - range: 337..338, - id: "x", - ctx: Load, + Named( + ExprNamed { + range: 336..343, + target: Starred( + ExprStarred { + range: 336..338, + value: Name( + ExprName { + range: 337..338, + id: "x", + ctx: Store, + }, + ), + ctx: Store, }, ), - ctx: Load, - }, - ), - NumberLiteral( - ExprNumberLiteral { - range: 342..343, - value: Int( - 2, + value: NumberLiteral( + ExprNumberLiteral { + range: 342..343, + value: Int( + 2, + ), + }, ), }, ), ], ctx: Load, - parenthesized: false, + parenthesized: true, }, ), }, @@ -1231,7 +1236,7 @@ Module( 8 | (*x if True else y, z, *x if True else y) 9 | (*lambda x: x, z, *lambda x: x) 10 | (*x := 2, z, *x := 2) - | ^^ Syntax Error: Starred expression cannot be used here + | ^^ Syntax Error: Assignment expression target must be an identifier | @@ -1239,34 +1244,7 @@ Module( 8 | (*x if True else y, z, *x if True else y) 9 | (*lambda x: x, z, *lambda x: x) 10 | (*x := 2, z, *x := 2) - | ^^ Syntax Error: Expected ')', found ':=' - | - - - | - 8 | (*x if True else y, z, *x if True else y) - 9 | (*lambda x: x, z, *lambda x: x) -10 | (*x := 2, z, *x := 2) - | ^^ Syntax Error: Expected ',', found ':=' - | - - - | - 8 | (*x if True else y, z, *x if True else y) - 9 | (*lambda x: x, z, *lambda x: x) -10 | (*x := 2, z, *x := 2) - | ^ Syntax Error: Expected a statement - | - - - | - 8 | (*x if True else y, z, *x if True else y) - 9 | (*lambda x: x, z, *lambda x: x) -10 | (*x := 2, z, *x := 2) - | ^ Syntax Error: Expected a statement -11 | -12 | -13 | # Non-parenthesized + | ^^ Syntax Error: Assignment expression target must be an identifier | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap index baa5df7880..978e113160 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__star_expression_precedence.py.snap @@ -339,24 +339,29 @@ Module( ExprSet { range: 186..198, elts: [ - Starred( - ExprStarred { - range: 187..189, - value: Name( - ExprName { - range: 188..189, - id: "x", - ctx: Load, + Named( + ExprNamed { + range: 187..194, + target: Starred( + ExprStarred { + range: 187..189, + value: Name( + ExprName { + range: 188..189, + id: "x", + ctx: Store, + }, + ), + ctx: Store, }, ), - ctx: Load, - }, - ), - NumberLiteral( - ExprNumberLiteral { - range: 193..194, - value: Int( - 2, + value: NumberLiteral( + ExprNumberLiteral { + range: 193..194, + value: Int( + 2, + ), + }, ), }, ), @@ -450,5 +455,5 @@ Module( 8 | {*x if True else y, z} 9 | {*lambda x: x, z} 10 | {*x := 2, z} - | ^^ Syntax Error: Expected ',', found ':=' + | ^^ Syntax Error: Assignment expression target must be an identifier | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_in_target_postfix_expr.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_in_target_postfix_expr.py.snap deleted file mode 100644 index c74d11d107..0000000000 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_in_target_postfix_expr.py.snap +++ /dev/null @@ -1,89 +0,0 @@ ---- -source: crates/ruff_python_parser/tests/fixtures.rs -input_file: crates/ruff_python_parser/resources/inline/err/for_in_target_postfix_expr.py ---- -## AST - -``` -Module( - ModModule { - range: 0..29, - body: [ - For( - StmtFor { - range: 0..28, - is_async: false, - target: Call( - ExprCall { - range: 4..13, - func: Name( - ExprName { - range: 4..5, - id: "d", - ctx: Load, - }, - ), - arguments: Arguments { - range: 5..13, - args: [ - Compare( - ExprCompare { - range: 6..12, - left: Name( - ExprName { - range: 6..7, - id: "x", - ctx: Load, - }, - ), - ops: [ - In, - ], - comparators: [ - Name( - ExprName { - range: 11..12, - id: "y", - ctx: Load, - }, - ), - ], - }, - ), - ], - keywords: [], - }, - }, - ), - iter: Name( - ExprName { - range: 17..23, - id: "target", - ctx: Load, - }, - ), - body: [ - Expr( - StmtExpr { - range: 25..28, - value: EllipsisLiteral( - ExprEllipsisLiteral { - range: 25..28, - }, - ), - }, - ), - ], - orelse: [], - }, - ), - ], - }, -) -``` -## Errors - - | -1 | for d(x in y) in target: ... - | ^^^^^^^^^ Syntax Error: Invalid assignment target - | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap index ae0cfbfda8..10e8692553 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target.py.snap @@ -7,7 +7,7 @@ input_file: crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_targ ``` Module( ModModule { - range: 0..132, + range: 0..154, body: [ For( StmtFor { @@ -233,22 +233,79 @@ Module( ), For( StmtFor { - range: 100..131, + range: 100..121, + is_async: false, + target: Yield( + ExprYield { + range: 104..116, + value: Some( + Compare( + ExprCompare { + range: 110..116, + left: Name( + ExprName { + range: 110..111, + id: "x", + ctx: Load, + }, + ), + ops: [ + In, + ], + comparators: [ + Name( + ExprName { + range: 115..116, + id: "y", + ctx: Load, + }, + ), + ], + }, + ), + ), + }, + ), + iter: Name( + ExprName { + range: 116..116, + id: "", + ctx: Invalid, + }, + ), + body: [ + Expr( + StmtExpr { + range: 118..121, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 118..121, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 122..153, is_async: false, target: List( ExprList { - range: 104..121, + range: 126..143, elts: [ Name( ExprName { - range: 105..106, + range: 127..128, id: "x", ctx: Store, }, ), NumberLiteral( ExprNumberLiteral { - range: 108..109, + range: 130..131, value: Int( 1, ), @@ -256,25 +313,25 @@ Module( ), Name( ExprName { - range: 111..112, + range: 133..134, id: "y", ctx: Store, }, ), Starred( ExprStarred { - range: 114..120, + range: 136..142, value: List( ExprList { - range: 115..120, + range: 137..142, elts: [ StringLiteral( ExprStringLiteral { - range: 116..119, + range: 138..141, value: StringLiteralValue { inner: Single( StringLiteral { - range: 116..119, + range: 138..141, value: "a", flags: StringLiteralFlags { quote_style: Double, @@ -299,7 +356,7 @@ Module( ), iter: Name( ExprName { - range: 125..126, + range: 147..148, id: "z", ctx: Load, }, @@ -307,10 +364,10 @@ Module( body: [ Expr( StmtExpr { - range: 128..131, + range: 150..153, value: EllipsisLiteral( ExprEllipsisLiteral { - range: 128..131, + range: 150..153, }, ), }, @@ -358,7 +415,7 @@ Module( 4 | for *x | y in z: ... | ^^^^^ Syntax Error: Invalid assignment target 5 | for await x in z: ... -6 | for [x, 1, y, *["a"]] in z: ... +6 | for yield x in y: ... | @@ -367,21 +424,40 @@ Module( 4 | for *x | y in z: ... 5 | for await x in z: ... | ^^^^^^^ Syntax Error: Invalid assignment target -6 | for [x, 1, y, *["a"]] in z: ... +6 | for yield x in y: ... +7 | for [x, 1, y, *["a"]] in z: ... | | 4 | for *x | y in z: ... 5 | for await x in z: ... -6 | for [x, 1, y, *["a"]] in z: ... +6 | for yield x in y: ... + | ^^^^^^^^^^^^ Syntax Error: Yield expression cannot be used here +7 | for [x, 1, y, *["a"]] in z: ... + | + + + | +4 | for *x | y in z: ... +5 | for await x in z: ... +6 | for yield x in y: ... + | ^ Syntax Error: Expected 'in', found ':' +7 | for [x, 1, y, *["a"]] in z: ... + | + + + | +5 | for await x in z: ... +6 | for yield x in y: ... +7 | for [x, 1, y, *["a"]] in z: ... | ^ Syntax Error: Invalid assignment target | | -4 | for *x | y in z: ... 5 | for await x in z: ... -6 | for [x, 1, y, *["a"]] in z: ... +6 | for yield x in y: ... +7 | for [x, 1, y, *["a"]] in z: ... | ^^^ Syntax Error: Invalid assignment target | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_binary_expr.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_binary_expr.py.snap new file mode 100644 index 0000000000..7d06a792e9 --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_binary_expr.py.snap @@ -0,0 +1,341 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target_binary_expr.py +--- +## AST + +``` +Module( + ModModule { + range: 0..124, + body: [ + For( + StmtFor { + range: 0..24, + is_async: false, + target: Compare( + ExprCompare { + range: 4..14, + left: Name( + ExprName { + range: 4..5, + id: "x", + ctx: Load, + }, + ), + ops: [ + NotIn, + ], + comparators: [ + Name( + ExprName { + range: 13..14, + id: "y", + ctx: Load, + }, + ), + ], + }, + ), + iter: Name( + ExprName { + range: 18..19, + id: "z", + ctx: Load, + }, + ), + body: [ + Expr( + StmtExpr { + range: 21..24, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 21..24, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 25..45, + is_async: false, + target: Compare( + ExprCompare { + range: 29..35, + left: Name( + ExprName { + range: 29..30, + id: "x", + ctx: Load, + }, + ), + ops: [ + Eq, + ], + comparators: [ + Name( + ExprName { + range: 34..35, + id: "y", + ctx: Load, + }, + ), + ], + }, + ), + iter: Name( + ExprName { + range: 39..40, + id: "z", + ctx: Load, + }, + ), + body: [ + Expr( + StmtExpr { + range: 42..45, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 42..45, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 46..66, + is_async: false, + target: BoolOp( + ExprBoolOp { + range: 50..56, + op: Or, + values: [ + Name( + ExprName { + range: 50..51, + id: "x", + ctx: Load, + }, + ), + Name( + ExprName { + range: 55..56, + id: "y", + ctx: Load, + }, + ), + ], + }, + ), + iter: Name( + ExprName { + range: 60..61, + id: "z", + ctx: Load, + }, + ), + body: [ + Expr( + StmtExpr { + range: 63..66, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 63..66, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 67..83, + is_async: false, + target: UnaryOp( + ExprUnaryOp { + range: 71..73, + op: USub, + operand: Name( + ExprName { + range: 72..73, + id: "x", + ctx: Store, + }, + ), + }, + ), + iter: Name( + ExprName { + range: 77..78, + id: "y", + ctx: Load, + }, + ), + body: [ + Expr( + StmtExpr { + range: 80..83, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 80..83, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 84..103, + is_async: false, + target: UnaryOp( + ExprUnaryOp { + range: 88..93, + op: Not, + operand: Name( + ExprName { + range: 92..93, + id: "x", + ctx: Store, + }, + ), + }, + ), + iter: Name( + ExprName { + range: 97..98, + id: "y", + ctx: Load, + }, + ), + body: [ + Expr( + StmtExpr { + range: 100..103, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 100..103, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 104..123, + is_async: false, + target: BinOp( + ExprBinOp { + range: 108..113, + left: Name( + ExprName { + range: 108..109, + id: "x", + ctx: Load, + }, + ), + op: BitOr, + right: Name( + ExprName { + range: 112..113, + id: "y", + ctx: Load, + }, + ), + }, + ), + iter: Name( + ExprName { + range: 117..118, + id: "z", + ctx: Load, + }, + ), + body: [ + Expr( + StmtExpr { + range: 120..123, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 120..123, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + ], + }, +) +``` +## Errors + + | +1 | for x not in y in z: ... + | ^^^^^^^^^^ Syntax Error: Invalid assignment target +2 | for x == y in z: ... +3 | for x or y in z: ... + | + + + | +1 | for x not in y in z: ... +2 | for x == y in z: ... + | ^^^^^^ Syntax Error: Invalid assignment target +3 | for x or y in z: ... +4 | for -x in y: ... + | + + + | +1 | for x not in y in z: ... +2 | for x == y in z: ... +3 | for x or y in z: ... + | ^^^^^^ Syntax Error: Invalid assignment target +4 | for -x in y: ... +5 | for not x in y: ... + | + + + | +2 | for x == y in z: ... +3 | for x or y in z: ... +4 | for -x in y: ... + | ^^ Syntax Error: Invalid assignment target +5 | for not x in y: ... +6 | for x | y in z: ... + | + + + | +3 | for x or y in z: ... +4 | for -x in y: ... +5 | for not x in y: ... + | ^^^^^ Syntax Error: Invalid assignment target +6 | for x | y in z: ... + | + + + | +4 | for -x in y: ... +5 | for not x in y: ... +6 | for x | y in z: ... + | ^^^^^ Syntax Error: Invalid assignment target + | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@parenthesized_compare_expr_in_for.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_in_keyword.py.snap similarity index 70% rename from crates/ruff_python_parser/tests/snapshots/invalid_syntax@parenthesized_compare_expr_in_for.py.snap rename to crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_in_keyword.py.snap index 53cde3a0c9..c6e9447da7 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@parenthesized_compare_expr_in_for.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@for_stmt_invalid_target_in_keyword.py.snap @@ -1,27 +1,95 @@ --- source: crates/ruff_python_parser/tests/fixtures.rs -input_file: crates/ruff_python_parser/resources/inline/err/parenthesized_compare_expr_in_for.py +input_file: crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target_in_keyword.py --- ## AST ``` Module( ModModule { - range: 0..141, + range: 0..170, body: [ For( StmtFor { - range: 0..27, + range: 0..28, is_async: false, target: Call( ExprCall { - range: 4..14, + range: 4..13, + func: Name( + ExprName { + range: 4..5, + id: "d", + ctx: Load, + }, + ), + arguments: Arguments { + range: 5..13, + args: [ + Compare( + ExprCompare { + range: 6..12, + left: Name( + ExprName { + range: 6..7, + id: "x", + ctx: Load, + }, + ), + ops: [ + In, + ], + comparators: [ + Name( + ExprName { + range: 11..12, + id: "y", + ctx: Load, + }, + ), + ], + }, + ), + ], + keywords: [], + }, + }, + ), + iter: Name( + ExprName { + range: 17..23, + id: "target", + ctx: Load, + }, + ), + body: [ + Expr( + StmtExpr { + range: 25..28, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 25..28, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 29..56, + is_async: false, + target: Call( + ExprCall { + range: 33..43, func: Compare( ExprCompare { - range: 5..11, + range: 34..40, left: Name( ExprName { - range: 5..6, + range: 34..35, id: "x", ctx: Load, }, @@ -32,7 +100,7 @@ Module( comparators: [ Name( ExprName { - range: 10..11, + range: 39..40, id: "y", ctx: Load, }, @@ -41,7 +109,7 @@ Module( }, ), arguments: Arguments { - range: 12..14, + range: 41..43, args: [], keywords: [], }, @@ -49,7 +117,7 @@ Module( ), iter: Name( ExprName { - range: 18..22, + range: 47..51, id: "iter", ctx: Load, }, @@ -57,10 +125,10 @@ Module( body: [ Expr( StmtExpr { - range: 24..27, + range: 53..56, value: EllipsisLiteral( ExprEllipsisLiteral { - range: 24..27, + range: 53..56, }, ), }, @@ -71,14 +139,14 @@ Module( ), For( StmtFor { - range: 28..53, + range: 57..82, is_async: false, target: Compare( ExprCompare { - range: 33..39, + range: 62..68, left: Name( ExprName { - range: 33..34, + range: 62..63, id: "x", ctx: Load, }, @@ -89,7 +157,7 @@ Module( comparators: [ Name( ExprName { - range: 38..39, + range: 67..68, id: "y", ctx: Load, }, @@ -97,72 +165,6 @@ Module( ], }, ), - iter: Name( - ExprName { - range: 44..48, - id: "iter", - ctx: Load, - }, - ), - body: [ - Expr( - StmtExpr { - range: 50..53, - value: EllipsisLiteral( - ExprEllipsisLiteral { - range: 50..53, - }, - ), - }, - ), - ], - orelse: [], - }, - ), - For( - StmtFor { - range: 54..82, - is_async: false, - target: Tuple( - ExprTuple { - range: 58..69, - elts: [ - Compare( - ExprCompare { - range: 59..65, - left: Name( - ExprName { - range: 59..60, - id: "x", - ctx: Load, - }, - ), - ops: [ - In, - ], - comparators: [ - Name( - ExprName { - range: 64..65, - id: "y", - ctx: Load, - }, - ), - ], - }, - ), - Name( - ExprName { - range: 67..68, - id: "z", - ctx: Store, - }, - ), - ], - ctx: Store, - parenthesized: true, - }, - ), iter: Name( ExprName { range: 73..77, @@ -189,8 +191,8 @@ Module( StmtFor { range: 83..111, is_async: false, - target: List( - ExprList { + target: Tuple( + ExprTuple { range: 87..98, elts: [ Compare( @@ -226,6 +228,7 @@ Module( ), ], ctx: Store, + parenthesized: true, }, ), iter: Name( @@ -254,8 +257,8 @@ Module( StmtFor { range: 112..140, is_async: false, - target: Set( - ExprSet { + target: List( + ExprList { range: 116..127, elts: [ Compare( @@ -286,10 +289,11 @@ Module( ExprName { range: 125..126, id: "z", - ctx: Load, + ctx: Store, }, ), ], + ctx: Store, }, ), iter: Name( @@ -314,6 +318,70 @@ Module( orelse: [], }, ), + For( + StmtFor { + range: 141..169, + is_async: false, + target: Set( + ExprSet { + range: 145..156, + elts: [ + Compare( + ExprCompare { + range: 146..152, + left: Name( + ExprName { + range: 146..147, + id: "x", + ctx: Load, + }, + ), + ops: [ + In, + ], + comparators: [ + Name( + ExprName { + range: 151..152, + id: "y", + ctx: Load, + }, + ), + ], + }, + ), + Name( + ExprName { + range: 154..155, + id: "z", + ctx: Load, + }, + ), + ], + }, + ), + iter: Name( + ExprName { + range: 160..164, + id: "iter", + ctx: Load, + }, + ), + body: [ + Expr( + StmtExpr { + range: 166..169, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 166..169, + }, + ), + }, + ), + ], + orelse: [], + }, + ), ], }, ) @@ -321,44 +389,54 @@ Module( ## Errors | -1 | for (x in y)() in iter: ... +1 | for d(x in y) in target: ... + | ^^^^^^^^^ Syntax Error: Invalid assignment target +2 | for (x in y)() in iter: ... +3 | for (x in y) in iter: ... + | + + + | +1 | for d(x in y) in target: ... +2 | for (x in y)() in iter: ... | ^^^^^^^^^^ Syntax Error: Invalid assignment target -2 | for (x in y) in iter: ... -3 | for (x in y, z) in iter: ... +3 | for (x in y) in iter: ... +4 | for (x in y, z) in iter: ... | | -1 | for (x in y)() in iter: ... -2 | for (x in y) in iter: ... +1 | for d(x in y) in target: ... +2 | for (x in y)() in iter: ... +3 | for (x in y) in iter: ... | ^^^^^^ Syntax Error: Invalid assignment target -3 | for (x in y, z) in iter: ... -4 | for [x in y, z] in iter: ... +4 | for (x in y, z) in iter: ... +5 | for [x in y, z] in iter: ... | | -1 | for (x in y)() in iter: ... -2 | for (x in y) in iter: ... -3 | for (x in y, z) in iter: ... +2 | for (x in y)() in iter: ... +3 | for (x in y) in iter: ... +4 | for (x in y, z) in iter: ... | ^^^^^^ Syntax Error: Invalid assignment target -4 | for [x in y, z] in iter: ... -5 | for {x in y, z} in iter: ... +5 | for [x in y, z] in iter: ... +6 | for {x in y, z} in iter: ... | | -2 | for (x in y) in iter: ... -3 | for (x in y, z) in iter: ... -4 | for [x in y, z] in iter: ... +3 | for (x in y) in iter: ... +4 | for (x in y, z) in iter: ... +5 | for [x in y, z] in iter: ... | ^^^^^^ Syntax Error: Invalid assignment target -5 | for {x in y, z} in iter: ... +6 | for {x in y, z} in iter: ... | | -3 | for (x in y, z) in iter: ... -4 | for [x in y, z] in iter: ... -5 | for {x in y, z} in iter: ... +4 | for (x in y, z) in iter: ... +5 | for [x in y, z] in iter: ... +6 | for {x in y, z} in iter: ... | ^^^^^^^^^^^ Syntax Error: Invalid assignment target | diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_postfix_expr.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_postfix_expr.py.snap deleted file mode 100644 index 13637f642d..0000000000 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_postfix_expr.py.snap +++ /dev/null @@ -1,78 +0,0 @@ ---- -source: crates/ruff_python_parser/tests/fixtures.rs -input_file: crates/ruff_python_parser/resources/inline/ok/for_in_target_postfix_expr.py ---- -## AST - -``` -Module( - ModModule { - range: 0..29, - body: [ - For( - StmtFor { - range: 0..28, - is_async: false, - target: Subscript( - ExprSubscript { - range: 4..13, - value: Name( - ExprName { - range: 4..5, - id: "d", - ctx: Load, - }, - ), - slice: Compare( - ExprCompare { - range: 6..12, - left: Name( - ExprName { - range: 6..7, - id: "x", - ctx: Load, - }, - ), - ops: [ - In, - ], - comparators: [ - Name( - ExprName { - range: 11..12, - id: "y", - ctx: Load, - }, - ), - ], - }, - ), - ctx: Store, - }, - ), - iter: Name( - ExprName { - range: 17..23, - id: "target", - ctx: Load, - }, - ), - body: [ - Expr( - StmtExpr { - range: 25..28, - value: EllipsisLiteral( - ExprEllipsisLiteral { - range: 25..28, - }, - ), - }, - ), - ], - orelse: [], - }, - ), - ], - }, -) -``` diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@parenthesized_compare_expr_in_for.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_valid_expr.py.snap similarity index 58% rename from crates/ruff_python_parser/tests/snapshots/valid_syntax@parenthesized_compare_expr_in_for.py.snap rename to crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_valid_expr.py.snap index d96d85bf19..04e9a2dd48 100644 --- a/crates/ruff_python_parser/tests/snapshots/valid_syntax@parenthesized_compare_expr_in_for.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@for_in_target_valid_expr.py.snap @@ -1,13 +1,13 @@ --- source: crates/ruff_python_parser/tests/fixtures.rs -input_file: crates/ruff_python_parser/resources/inline/ok/parenthesized_compare_expr_in_for.py +input_file: crates/ruff_python_parser/resources/inline/ok/for_in_target_valid_expr.py --- ## AST ``` Module( ModModule { - range: 0..60, + range: 0..89, body: [ For( StmtFor { @@ -15,13 +15,20 @@ Module( is_async: false, target: Subscript( ExprSubscript { - range: 4..15, - value: Compare( + range: 4..13, + value: Name( + ExprName { + range: 4..5, + id: "d", + ctx: Load, + }, + ), + slice: Compare( ExprCompare { - range: 5..11, + range: 6..12, left: Name( ExprName { - range: 5..6, + range: 6..7, id: "x", ctx: Load, }, @@ -32,7 +39,7 @@ Module( comparators: [ Name( ExprName { - range: 10..11, + range: 11..12, id: "y", ctx: Load, }, @@ -40,21 +47,13 @@ Module( ], }, ), - slice: NumberLiteral( - ExprNumberLiteral { - range: 13..14, - value: Int( - 0, - ), - }, - ), ctx: Store, }, ), iter: Name( ExprName { - range: 19..23, - id: "iter", + range: 17..23, + id: "target", ctx: Load, }, ), @@ -75,11 +74,11 @@ Module( ), For( StmtFor { - range: 29..59, + range: 29..57, is_async: false, - target: Attribute( - ExprAttribute { - range: 33..46, + target: Subscript( + ExprSubscript { + range: 33..44, value: Compare( ExprCompare { range: 34..40, @@ -104,16 +103,20 @@ Module( ], }, ), - attr: Identifier { - id: "attr", - range: 42..46, - }, + slice: NumberLiteral( + ExprNumberLiteral { + range: 42..43, + value: Int( + 0, + ), + }, + ), ctx: Store, }, ), iter: Name( ExprName { - range: 50..54, + range: 48..52, id: "iter", ctx: Load, }, @@ -121,10 +124,70 @@ Module( body: [ Expr( StmtExpr { - range: 56..59, + range: 54..57, value: EllipsisLiteral( ExprEllipsisLiteral { - range: 56..59, + range: 54..57, + }, + ), + }, + ), + ], + orelse: [], + }, + ), + For( + StmtFor { + range: 58..88, + is_async: false, + target: Attribute( + ExprAttribute { + range: 62..75, + value: Compare( + ExprCompare { + range: 63..69, + left: Name( + ExprName { + range: 63..64, + id: "x", + ctx: Load, + }, + ), + ops: [ + In, + ], + comparators: [ + Name( + ExprName { + range: 68..69, + id: "y", + ctx: Load, + }, + ), + ], + }, + ), + attr: Identifier { + id: "attr", + range: 71..75, + }, + ctx: Store, + }, + ), + iter: Name( + ExprName { + range: 79..83, + id: "iter", + ctx: Load, + }, + ), + body: [ + Expr( + StmtExpr { + range: 85..88, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 85..88, }, ), },