From 474b00568ad78f02ad8e19b8166cbeb6d69f8511 Mon Sep 17 00:00:00 2001 From: Dan Parizher <105245560+danparizher@users.noreply.github.com> Date: Mon, 24 Nov 2025 00:40:27 -0500 Subject: [PATCH] [`parser`] Fix panic when parsing IPython escape command expressions (#21480) ## Summary Fixes a panic when parsing IPython escape commands with `Help` kind (`?`) in expression contexts. The parser now reports an error instead of panicking. Fixes #21465. ## Problem The parser panicked with `unreachable!()` in `parse_ipython_escape_command_expression` when encountering escape commands with `Help` kind (`?`) in expression contexts, where only `Magic` (`%`) and `Shell` (`!`) are allowed. ## Approach Replaced the `unreachable!()` panic with error handling that adds a `ParseErrorType::OtherError` and continues parsing, returning a valid AST node with the error attached. ## Test Plan Added `test_ipython_escape_command_in_with_statement` and `test_ipython_help_escape_command_as_expression` to verify the fix. --------- Co-authored-by: Dhruv Manilawala --- ...on_help_escape_command_error_recovery_1.py | 3 + ...on_help_escape_command_error_recovery_2.py | 3 + ...on_help_escape_command_error_recovery_3.py | 4 + crates/ruff_python_parser/src/parser/mod.rs | 22 +++- ...lp_escape_command_error_recovery_1.py.snap | 84 +++++++++++++ ...lp_escape_command_error_recovery_2.py.snap | 80 +++++++++++++ ...lp_escape_command_error_recovery_3.py.snap | 112 ++++++++++++++++++ 7 files changed, 307 insertions(+), 1 deletion(-) create mode 100644 crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_1.py create mode 100644 crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_2.py create mode 100644 crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_3.py create mode 100644 crates/ruff_python_parser/tests/snapshots/invalid_syntax@ipython_help_escape_command_error_recovery_1.py.snap create mode 100644 crates/ruff_python_parser/tests/snapshots/invalid_syntax@ipython_help_escape_command_error_recovery_2.py.snap create mode 100644 crates/ruff_python_parser/tests/snapshots/invalid_syntax@ipython_help_escape_command_error_recovery_3.py.snap diff --git a/crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_1.py b/crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_1.py new file mode 100644 index 0000000000..7131abb471 --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_1.py @@ -0,0 +1,3 @@ +# parse_options: {"mode": "ipython"} +with (a, ?b) +? diff --git a/crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_2.py b/crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_2.py new file mode 100644 index 0000000000..689486abb4 --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_2.py @@ -0,0 +1,3 @@ +# parse_options: {"mode": "ipython"} +with (a, ?b +? diff --git a/crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_3.py b/crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_3.py new file mode 100644 index 0000000000..31d87f41e7 --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_3.py @@ -0,0 +1,4 @@ +# parse_options: {"mode": "ipython"} +with a, ?b +? +x = 1 diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index 5f35b84b04..90396cb72d 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -1115,7 +1115,27 @@ impl RecoveryContextKind { TokenKind::Colon => Some(ListTerminatorKind::ErrorRecovery), _ => None, }, - WithItemKind::Unparenthesized | WithItemKind::ParenthesizedExpression => p + // test_err ipython_help_escape_command_error_recovery_1 + // # parse_options: {"mode": "ipython"} + // with (a, ?b) + // ? + + // test_err ipython_help_escape_command_error_recovery_2 + // # parse_options: {"mode": "ipython"} + // with (a, ?b + // ? + + // test_err ipython_help_escape_command_error_recovery_3 + // # parse_options: {"mode": "ipython"} + // with a, ?b + // ? + // x = 1 + WithItemKind::Unparenthesized => matches!( + p.current_token_kind(), + TokenKind::Colon | TokenKind::Newline + ) + .then_some(ListTerminatorKind::Regular), + WithItemKind::ParenthesizedExpression => p .at(TokenKind::Colon) .then_some(ListTerminatorKind::Regular), }, diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ipython_help_escape_command_error_recovery_1.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ipython_help_escape_command_error_recovery_1.py.snap new file mode 100644 index 0000000000..aa33df6cbc --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ipython_help_escape_command_error_recovery_1.py.snap @@ -0,0 +1,84 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_1.py +--- +## AST + +``` +Module( + ModModule { + node_index: NodeIndex(None), + range: 0..52, + body: [ + With( + StmtWith { + node_index: NodeIndex(None), + range: 37..49, + is_async: false, + items: [ + WithItem { + range: 43..44, + node_index: NodeIndex(None), + context_expr: Name( + ExprName { + node_index: NodeIndex(None), + range: 43..44, + id: Name("a"), + ctx: Load, + }, + ), + optional_vars: None, + }, + WithItem { + range: 47..48, + node_index: NodeIndex(None), + context_expr: Name( + ExprName { + node_index: NodeIndex(None), + range: 47..48, + id: Name("b"), + ctx: Load, + }, + ), + optional_vars: None, + }, + ], + body: [], + }, + ), + IpyEscapeCommand( + StmtIpyEscapeCommand { + node_index: NodeIndex(None), + range: 50..51, + kind: Help, + value: "", + }, + ), + ], + }, +) +``` +## Errors + + | +1 | # parse_options: {"mode": "ipython"} +2 | with (a, ?b) + | ^ Syntax Error: Expected `,`, found `?` +3 | ? + | + + + | +1 | # parse_options: {"mode": "ipython"} +2 | with (a, ?b) + | ^ Syntax Error: Expected `:`, found newline +3 | ? + | + + + | +1 | # parse_options: {"mode": "ipython"} +2 | with (a, ?b) +3 | ? + | ^ Syntax Error: Expected an indented block after `with` statement + | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ipython_help_escape_command_error_recovery_2.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ipython_help_escape_command_error_recovery_2.py.snap new file mode 100644 index 0000000000..424b43d107 --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ipython_help_escape_command_error_recovery_2.py.snap @@ -0,0 +1,80 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_2.py +--- +## AST + +``` +Module( + ModModule { + node_index: NodeIndex(None), + range: 0..51, + body: [ + With( + StmtWith { + node_index: NodeIndex(None), + range: 37..51, + is_async: false, + items: [ + WithItem { + range: 42..51, + node_index: NodeIndex(None), + context_expr: Tuple( + ExprTuple { + node_index: NodeIndex(None), + range: 42..51, + elts: [ + Name( + ExprName { + node_index: NodeIndex(None), + range: 43..44, + id: Name("a"), + ctx: Load, + }, + ), + Name( + ExprName { + node_index: NodeIndex(None), + range: 47..48, + id: Name("b"), + ctx: Load, + }, + ), + ], + ctx: Load, + parenthesized: true, + }, + ), + optional_vars: None, + }, + ], + body: [], + }, + ), + ], + }, +) +``` +## Errors + + | +1 | # parse_options: {"mode": "ipython"} +2 | with (a, ?b + | ^ Syntax Error: Expected an expression or a ')' +3 | ? + | + + + | +1 | # parse_options: {"mode": "ipython"} +2 | with (a, ?b +3 | ? + | ^ Syntax Error: Expected `,`, found `?` + | + + + | +2 | with (a, ?b +3 | ? + | ^ Syntax Error: unexpected EOF while parsing + | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ipython_help_escape_command_error_recovery_3.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ipython_help_escape_command_error_recovery_3.py.snap new file mode 100644 index 0000000000..33079dbc60 --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@ipython_help_escape_command_error_recovery_3.py.snap @@ -0,0 +1,112 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/err/ipython_help_escape_command_error_recovery_3.py +--- +## AST + +``` +Module( + ModModule { + node_index: NodeIndex(None), + range: 0..56, + body: [ + With( + StmtWith { + node_index: NodeIndex(None), + range: 37..47, + is_async: false, + items: [ + WithItem { + range: 42..43, + node_index: NodeIndex(None), + context_expr: Name( + ExprName { + node_index: NodeIndex(None), + range: 42..43, + id: Name("a"), + ctx: Load, + }, + ), + optional_vars: None, + }, + WithItem { + range: 46..47, + node_index: NodeIndex(None), + context_expr: Name( + ExprName { + node_index: NodeIndex(None), + range: 46..47, + id: Name("b"), + ctx: Load, + }, + ), + optional_vars: None, + }, + ], + body: [], + }, + ), + IpyEscapeCommand( + StmtIpyEscapeCommand { + node_index: NodeIndex(None), + range: 48..49, + kind: Help, + value: "", + }, + ), + Assign( + StmtAssign { + node_index: NodeIndex(None), + range: 50..55, + targets: [ + Name( + ExprName { + node_index: NodeIndex(None), + range: 50..51, + id: Name("x"), + ctx: Store, + }, + ), + ], + value: NumberLiteral( + ExprNumberLiteral { + node_index: NodeIndex(None), + range: 54..55, + value: Int( + 1, + ), + }, + ), + }, + ), + ], + }, +) +``` +## Errors + + | +1 | # parse_options: {"mode": "ipython"} +2 | with a, ?b + | ^ Syntax Error: Expected `,`, found `?` +3 | ? +4 | x = 1 + | + + + | +1 | # parse_options: {"mode": "ipython"} +2 | with a, ?b + | ^ Syntax Error: Expected `:`, found newline +3 | ? +4 | x = 1 + | + + + | +1 | # parse_options: {"mode": "ipython"} +2 | with a, ?b +3 | ? + | ^ Syntax Error: Expected an indented block after `with` statement +4 | x = 1 + |