From 28cc71fb6bc967fe04ec3a620192e7b8bda513c2 Mon Sep 17 00:00:00 2001 From: Dhruv Manilawala Date: Tue, 7 May 2024 14:54:57 +0530 Subject: [PATCH] Remove cyclic dev dependency with the parser crate (#11261) ## Summary This PR removes the cyclic dev dependency some of the crates had with the parser crate. The cyclic dependencies are: * `ruff_python_ast` has a **dev dependency** on `ruff_python_parser` and `ruff_python_parser` directly depends on `ruff_python_ast` * `ruff_python_trivia` has a **dev dependency** on `ruff_python_parser` and `ruff_python_parser` has an indirect dependency on `ruff_python_trivia` (`ruff_python_parser` - `ruff_python_ast` - `ruff_python_trivia`) Specifically, this PR does the following: * Introduce two new crates * `ruff_python_ast_integration_tests` and move the tests from the `ruff_python_ast` crate which uses the parser in this crate * `ruff_python_trivia_integration_tests` and move the tests from the `ruff_python_trivia` crate which uses the parser in this crate ### Motivation The main motivation for this PR is to help development. Before this PR, `rust-analyzer` wouldn't provide any intellisense in the `ruff_python_parser` crate regarding the symbols in `ruff_python_ast` crate. ``` [ERROR][2024-05-03 13:47:06] .../vim/lsp/rpc.lua:770 "rpc" "/Users/dhruv/.cargo/bin/rust-analyzer" "stderr" "[ERROR project_model::workspace] cyclic deps: ruff_python_parser(Idx::(50)) -> ruff_python_ast(Idx::(37)), alternative path: ruff_python_ast(Idx::(37)) -> ruff_python_parser(Idx::(50))\n" ``` ## Test Plan Check the logs of `rust-analyzer` to not see any signs of cyclic dependency. --- Cargo.lock | 26 +- crates/ruff_python_ast/Cargo.toml | 1 - .../Cargo.toml | 23 + .../README.md | 13 + .../src/lib.rs | 1 + .../tests/identifier.rs | 3 +- .../tests/parenthesize.rs | 0 .../tests/preorder.rs | 0 .../snapshots/preorder__bytes_literals.snap | 3 +- .../preorder__class_type_parameters.snap | 3 +- .../tests/snapshots/preorder__compare.snap | 3 +- .../tests/snapshots/preorder__decorators.snap | 3 +- .../preorder__dict_comprehension.snap | 3 +- .../tests/snapshots/preorder__f_strings.snap | 3 +- .../preorder__function_arguments.snap | 3 +- ...function_positional_only_with_default.snap | 3 +- .../preorder__function_type_parameters.snap | 3 +- .../preorder__list_comprehension.snap | 3 +- .../preorder__match_class_pattern.snap | 3 +- .../preorder__set_comprehension.snap | 3 +- .../snapshots/preorder__string_literals.snap | 3 +- .../snapshots/preorder__type_aliases.snap | 3 +- .../snapshots/visitor__bytes_literals.snap | 3 +- .../visitor__class_type_parameters.snap | 3 +- .../tests/snapshots/visitor__compare.snap | 3 +- .../tests/snapshots/visitor__decorators.snap | 3 +- .../visitor__dict_comprehension.snap | 3 +- .../tests/snapshots/visitor__f_strings.snap | 3 +- .../visitor__function_arguments.snap | 3 +- ...function_positional_only_with_default.snap | 3 +- .../visitor__function_type_parameters.snap | 3 +- .../visitor__list_comprehension.snap | 3 +- .../visitor__match_class_pattern.snap | 3 +- .../snapshots/visitor__set_comprehension.snap | 3 +- .../snapshots/visitor__string_literals.snap | 3 +- .../snapshots/visitor__type_aliases.snap | 3 +- .../tests/stmt_if.rs | 1 - .../tests/visitor.rs | 12 +- crates/ruff_python_trivia/Cargo.toml | 2 - .../ruff_python_trivia/src/comment_ranges.rs | 155 ------- crates/ruff_python_trivia/src/tokenizer.rs | 423 ------------------ crates/ruff_python_trivia/src/whitespace.rs | 48 -- .../Cargo.toml | 24 + .../README.md | 13 + .../src/lib.rs | 1 + .../tests/block_comments.rs | 151 +++++++ .../tests/simple_tokenizer.rs | 417 +++++++++++++++++ .../snapshots/simple_tokenizer__Reverse.snap} | 2 +- ...ment_containing_single_quoted_string.snap} | 2 +- ...ment_containing_triple_quoted_string.snap} | 2 +- ...mple_tokenizer__empty_string_literal.snap} | 2 +- ..._identifier_ending_in_non_start_char.snap} | 2 +- ...identifier_starting_with_string_kind.snap} | 2 +- ...e_word_with_only_id_continuing_chars.snap} | 2 +- ..._multiline_string_containing_comment.snap} | 2 +- ...tiline_string_implicit_concatenation.snap} | 2 +- ...string_followed_by_multiple_comments.snap} | 2 +- ...ple_tokenizer__string_with_byte_kind.snap} | 2 +- ...string_with_double_escaped_backslash.snap} | 2 +- ...tokenizer__string_with_escaped_quote.snap} | 2 +- ..._tokenizer__string_with_invalid_kind.snap} | 2 +- .../simple_tokenizer__string_with_kind.snap} | 2 +- .../simple_tokenizer__tokenize_bogus.snap} | 2 +- .../simple_tokenizer__tokenize_comma.snap} | 2 +- ...ple_tokenizer__tokenize_continuation.snap} | 2 +- .../simple_tokenizer__tokenize_eq.snap} | 2 +- ...okenizer__tokenize_invalid_operators.snap} | 2 +- ...simple_tokenizer__tokenize_multichar.snap} | 2 +- .../simple_tokenizer__tokenize_not_eq.snap} | 2 +- ...simple_tokenizer__tokenize_operators.snap} | 2 +- ...mple_tokenizer__tokenize_parentheses.snap} | 2 +- .../simple_tokenizer__tokenize_slash.snap} | 2 +- ...simple_tokenizer__tokenize_substring.snap} | 2 +- .../simple_tokenizer__tokenize_trivia.snap} | 2 +- .../simple_tokenizer__tricky_unicode.snap} | 2 +- ..._multiline_string_containing_comment.snap} | 2 +- .../tests/whitespace.rs | 43 ++ crates/ruff_source_file/Cargo.toml | 3 +- 78 files changed, 774 insertions(+), 728 deletions(-) create mode 100644 crates/ruff_python_ast_integration_tests/Cargo.toml create mode 100644 crates/ruff_python_ast_integration_tests/README.md create mode 100644 crates/ruff_python_ast_integration_tests/src/lib.rs rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/identifier.rs (99%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/parenthesize.rs (100%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/preorder.rs (100%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__bytes_literals.snap (67%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__class_type_parameters.snap (78%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__compare.snap (70%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__decorators.snap (73%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__dict_comprehension.snap (77%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__f_strings.snap (84%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__function_arguments.snap (88%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__function_positional_only_with_default.snap (82%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__function_type_parameters.snap (79%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__list_comprehension.snap (68%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__match_class_pattern.snap (91%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__set_comprehension.snap (68%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__string_literals.snap (67%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/preorder__type_aliases.snap (80%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__bytes_literals.snap (63%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__class_type_parameters.snap (73%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__compare.snap (67%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__decorators.snap (65%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__dict_comprehension.snap (74%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__f_strings.snap (82%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__function_arguments.snap (80%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__function_positional_only_with_default.snap (73%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__function_type_parameters.snap (75%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__list_comprehension.snap (65%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__match_class_pattern.snap (88%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__set_comprehension.snap (64%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__string_literals.snap (64%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/snapshots/visitor__type_aliases.snap (76%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/stmt_if.rs (99%) rename crates/{ruff_python_ast => ruff_python_ast_integration_tests}/tests/visitor.rs (96%) create mode 100644 crates/ruff_python_trivia_integration_tests/Cargo.toml create mode 100644 crates/ruff_python_trivia_integration_tests/README.md create mode 100644 crates/ruff_python_trivia_integration_tests/src/lib.rs create mode 100644 crates/ruff_python_trivia_integration_tests/tests/block_comments.rs create mode 100644 crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__Reverse.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__Reverse.snap} (76%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_single_quoted_string.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__comment_containing_single_quoted_string.snap} (81%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_triple_quoted_string.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__comment_containing_triple_quoted_string.snap} (81%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__empty_string_literal.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__empty_string_literal.snap} (80%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__identifier_ending_in_non_start_char.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__identifier_ending_in_non_start_char.snap} (58%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__identifier_starting_with_string_kind.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__identifier_starting_with_string_kind.snap} (76%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__ignore_word_with_only_id_continuing_chars.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__ignore_word_with_only_id_continuing_chars.snap} (69%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_containing_comment.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__single_quoted_multiline_string_containing_comment.snap} (70%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_implicit_concatenation.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__single_quoted_multiline_string_implicit_concatenation.snap} (70%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_followed_by_multiple_comments.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_followed_by_multiple_comments.snap} (81%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_byte_kind.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_byte_kind.snap} (69%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_double_escaped_backslash.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_double_escaped_backslash.snap} (81%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_escaped_quote.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_escaped_quote.snap} (81%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_invalid_kind.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_invalid_kind.snap} (75%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_kind.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_kind.snap} (69%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_bogus.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_bogus.snap} (83%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_comma.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_comma.snap} (79%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_continuation.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_continuation.snap} (85%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_eq.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_eq.snap} (69%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_invalid_operators.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_invalid_operators.snap} (80%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_multichar.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_multichar.snap} (87%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_not_eq.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_not_eq.snap} (69%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_operators.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_operators.snap} (95%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_parentheses.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_parentheses.snap} (85%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_slash.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_slash.snap} (89%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_substring.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_substring.snap} (76%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_trivia.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_trivia.snap} (80%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tricky_unicode.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tricky_unicode.snap} (58%) rename crates/{ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__triple_quoted_multiline_string_containing_comment.snap => ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__triple_quoted_multiline_string_containing_comment.snap} (70%) create mode 100644 crates/ruff_python_trivia_integration_tests/tests/whitespace.rs diff --git a/Cargo.lock b/Cargo.lock index 40eaa65c2f..aba89c1598 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2204,7 +2204,6 @@ dependencies = [ "is-macro", "itertools 0.12.1", "once_cell", - "ruff_python_parser", "ruff_python_trivia", "ruff_source_file", "ruff_text_size", @@ -2212,6 +2211,17 @@ dependencies = [ "serde", ] +[[package]] +name = "ruff_python_ast_integration_tests" +version = "0.0.0" +dependencies = [ + "insta", + "ruff_python_ast", + "ruff_python_parser", + "ruff_python_trivia", + "ruff_text_size", +] + [[package]] name = "ruff_python_codegen" version = "0.0.0" @@ -2340,13 +2350,23 @@ version = "0.0.0" dependencies = [ "insta", "itertools 0.12.1", - "ruff_python_index", - "ruff_python_parser", "ruff_source_file", "ruff_text_size", "unicode-ident", ] +[[package]] +name = "ruff_python_trivia_integration_tests" +version = "0.0.0" +dependencies = [ + "insta", + "ruff_python_index", + "ruff_python_parser", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", +] + [[package]] name = "ruff_server" version = "0.2.2" diff --git a/crates/ruff_python_ast/Cargo.toml b/crates/ruff_python_ast/Cargo.toml index 6918e6aed9..675deb7b2e 100644 --- a/crates/ruff_python_ast/Cargo.toml +++ b/crates/ruff_python_ast/Cargo.toml @@ -27,7 +27,6 @@ serde = { workspace = true, optional = true } [dev-dependencies] insta = { workspace = true } -ruff_python_parser = { path = "../ruff_python_parser" } [features] serde = ["dep:serde", "ruff_text_size/serde"] diff --git a/crates/ruff_python_ast_integration_tests/Cargo.toml b/crates/ruff_python_ast_integration_tests/Cargo.toml new file mode 100644 index 0000000000..f8c8efc6cd --- /dev/null +++ b/crates/ruff_python_ast_integration_tests/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "ruff_python_ast_integration_tests" +version = "0.0.0" +edition.workspace = true +rust-version.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] + +[dev-dependencies] +ruff_python_parser = { path = "../ruff_python_parser" } +ruff_python_ast = { path = "../ruff_python_ast" } +ruff_python_trivia = { path = "../ruff_python_trivia" } +ruff_text_size = { path = "../ruff_text_size" } + +insta = { workspace = true } + +[lints] +workspace = true diff --git a/crates/ruff_python_ast_integration_tests/README.md b/crates/ruff_python_ast_integration_tests/README.md new file mode 100644 index 0000000000..68ac4e4298 --- /dev/null +++ b/crates/ruff_python_ast_integration_tests/README.md @@ -0,0 +1,13 @@ +# Integration tests for `ruff_python_ast` + +This crate includes integration tests for the `ruff_python_ast` crate. + +The reason for having a separate crate is to avoid introducing a dev circular +dependency between the `ruff_python_parser` crate and the `ruff_python_ast` crate. + +This crate shouldn't include any code, only tests. + +**Reference:** + +- `rust-analyzer` issue: +- Ruff's pull request: diff --git a/crates/ruff_python_ast_integration_tests/src/lib.rs b/crates/ruff_python_ast_integration_tests/src/lib.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/ruff_python_ast_integration_tests/src/lib.rs @@ -0,0 +1 @@ + diff --git a/crates/ruff_python_ast/tests/identifier.rs b/crates/ruff_python_ast_integration_tests/tests/identifier.rs similarity index 99% rename from crates/ruff_python_ast/tests/identifier.rs rename to crates/ruff_python_ast_integration_tests/tests/identifier.rs index aaececf89c..1e70c4fd65 100644 --- a/crates/ruff_python_ast/tests/identifier.rs +++ b/crates/ruff_python_ast_integration_tests/tests/identifier.rs @@ -1,8 +1,7 @@ +use ruff_python_ast::identifier; use ruff_python_parser::{parse_suite, ParseError}; use ruff_text_size::{TextRange, TextSize}; -use ruff_python_ast::identifier; - #[test] fn extract_else_range() -> Result<(), ParseError> { let contents = r" diff --git a/crates/ruff_python_ast/tests/parenthesize.rs b/crates/ruff_python_ast_integration_tests/tests/parenthesize.rs similarity index 100% rename from crates/ruff_python_ast/tests/parenthesize.rs rename to crates/ruff_python_ast_integration_tests/tests/parenthesize.rs diff --git a/crates/ruff_python_ast/tests/preorder.rs b/crates/ruff_python_ast_integration_tests/tests/preorder.rs similarity index 100% rename from crates/ruff_python_ast/tests/preorder.rs rename to crates/ruff_python_ast_integration_tests/tests/preorder.rs diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__bytes_literals.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__bytes_literals.snap similarity index 67% rename from crates/ruff_python_ast/tests/snapshots/preorder__bytes_literals.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__bytes_literals.snap index d71ea07e19..11df74981c 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__bytes_literals.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__bytes_literals.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -8,4 +8,3 @@ expression: trace - BytesLiteral - BytesLiteral - BytesLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__class_type_parameters.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__class_type_parameters.snap similarity index 78% rename from crates/ruff_python_ast/tests/snapshots/preorder__class_type_parameters.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__class_type_parameters.snap index 20ef6b71fa..7b72f072ca 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__class_type_parameters.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__class_type_parameters.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -12,4 +12,3 @@ expression: trace - TypeParamParamSpec - StmtExpr - ExprEllipsisLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__compare.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__compare.snap similarity index 70% rename from crates/ruff_python_ast/tests/snapshots/preorder__compare.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__compare.snap index 848886f767..7982572267 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__compare.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__compare.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -10,4 +10,3 @@ expression: trace - ExprName - Lt - ExprNumberLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__decorators.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__decorators.snap similarity index 73% rename from crates/ruff_python_ast/tests/snapshots/preorder__decorators.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__decorators.snap index e91bd2e83b..f6b6c175ae 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__decorators.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__decorators.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -12,4 +12,3 @@ expression: trace - Decorator - ExprName - StmtPass - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__dict_comprehension.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__dict_comprehension.snap similarity index 77% rename from crates/ruff_python_ast/tests/snapshots/preorder__dict_comprehension.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__dict_comprehension.snap index d1479d6fed..2fa8d035d3 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__dict_comprehension.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__dict_comprehension.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -13,4 +13,3 @@ expression: trace - Comprehension - ExprName - ExprName - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__f_strings.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__f_strings.snap similarity index 84% rename from crates/ruff_python_ast/tests/snapshots/preorder__f_strings.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__f_strings.snap index 043c58064b..296252a266 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__f_strings.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__f_strings.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -15,4 +15,3 @@ expression: trace - ExprName - FStringLiteralElement - FStringLiteralElement - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__function_arguments.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__function_arguments.snap similarity index 88% rename from crates/ruff_python_ast/tests/snapshots/preorder__function_arguments.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__function_arguments.snap index da83fc10f8..d743ba0fa2 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__function_arguments.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__function_arguments.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -23,4 +23,3 @@ expression: trace - ExprNumberLiteral - Parameter - StmtPass - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__function_positional_only_with_default.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__function_positional_only_with_default.snap similarity index 82% rename from crates/ruff_python_ast/tests/snapshots/preorder__function_positional_only_with_default.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__function_positional_only_with_default.snap index 6511aa3468..198e3b2aa3 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__function_positional_only_with_default.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__function_positional_only_with_default.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -15,4 +15,3 @@ expression: trace - ExprNumberLiteral - Parameter - StmtPass - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__function_type_parameters.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__function_type_parameters.snap similarity index 79% rename from crates/ruff_python_ast/tests/snapshots/preorder__function_type_parameters.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__function_type_parameters.snap index 11b98d0fa7..d30516bf7b 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__function_type_parameters.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__function_type_parameters.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -13,4 +13,3 @@ expression: trace - Parameters - StmtExpr - ExprEllipsisLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__list_comprehension.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__list_comprehension.snap similarity index 68% rename from crates/ruff_python_ast/tests/snapshots/preorder__list_comprehension.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__list_comprehension.snap index dc15fb0197..d7fa6bda2f 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__list_comprehension.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__list_comprehension.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -9,4 +9,3 @@ expression: trace - Comprehension - ExprName - ExprName - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__match_class_pattern.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__match_class_pattern.snap similarity index 91% rename from crates/ruff_python_ast/tests/snapshots/preorder__match_class_pattern.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__match_class_pattern.snap index dbe56f11ff..70fa1276fe 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__match_class_pattern.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__match_class_pattern.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -30,4 +30,3 @@ expression: trace - ExprNumberLiteral - StmtExpr - ExprEllipsisLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__set_comprehension.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__set_comprehension.snap similarity index 68% rename from crates/ruff_python_ast/tests/snapshots/preorder__set_comprehension.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__set_comprehension.snap index e9abdf1245..47c7344d61 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__set_comprehension.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__set_comprehension.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -9,4 +9,3 @@ expression: trace - Comprehension - ExprName - ExprName - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__string_literals.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__string_literals.snap similarity index 67% rename from crates/ruff_python_ast/tests/snapshots/preorder__string_literals.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__string_literals.snap index 7b62ce78b7..fae44b6ed9 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__string_literals.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__string_literals.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -8,4 +8,3 @@ expression: trace - StringLiteral - StringLiteral - StringLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/preorder__type_aliases.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__type_aliases.snap similarity index 80% rename from crates/ruff_python_ast/tests/snapshots/preorder__type_aliases.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__type_aliases.snap index 56d6aec5c0..5c986912e2 100644 --- a/crates/ruff_python_ast/tests/snapshots/preorder__type_aliases.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__type_aliases.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/preorder.rs +source: crates/ruff_python_ast_integration_tests/tests/preorder.rs expression: trace --- - ModModule @@ -14,4 +14,3 @@ expression: trace - ExprSubscript - ExprName - ExprName - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__bytes_literals.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__bytes_literals.snap similarity index 63% rename from crates/ruff_python_ast/tests/snapshots/visitor__bytes_literals.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__bytes_literals.snap index 57bed3a9c7..0ad5f4f890 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__bytes_literals.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__bytes_literals.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtExpr @@ -7,4 +7,3 @@ expression: trace - BytesLiteral - BytesLiteral - BytesLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__class_type_parameters.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__class_type_parameters.snap similarity index 73% rename from crates/ruff_python_ast/tests/snapshots/visitor__class_type_parameters.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__class_type_parameters.snap index 5ca38f249f..cf16702250 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__class_type_parameters.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__class_type_parameters.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtClassDef @@ -10,4 +10,3 @@ expression: trace - TypeParamParamSpec - StmtExpr - ExprEllipsisLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__compare.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__compare.snap similarity index 67% rename from crates/ruff_python_ast/tests/snapshots/visitor__compare.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__compare.snap index 6a5f99eb4d..8c0425fa43 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__compare.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__compare.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtExpr @@ -9,4 +9,3 @@ expression: trace - Lt - ExprName - ExprNumberLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__decorators.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__decorators.snap similarity index 65% rename from crates/ruff_python_ast/tests/snapshots/visitor__decorators.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__decorators.snap index c633132e04..799385941d 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__decorators.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__decorators.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtFunctionDef @@ -9,4 +9,3 @@ expression: trace - StmtClassDef - ExprName - StmtPass - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__dict_comprehension.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__dict_comprehension.snap similarity index 74% rename from crates/ruff_python_ast/tests/snapshots/visitor__dict_comprehension.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__dict_comprehension.snap index 138cb45b38..40fe6e72eb 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__dict_comprehension.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__dict_comprehension.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtExpr @@ -12,4 +12,3 @@ expression: trace - ExprName - Pow - ExprNumberLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__f_strings.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__f_strings.snap similarity index 82% rename from crates/ruff_python_ast/tests/snapshots/visitor__f_strings.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__f_strings.snap index a5c8a8b905..2e9b7dccbc 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__f_strings.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__f_strings.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtExpr @@ -14,4 +14,3 @@ expression: trace - ExprName - FStringLiteralElement - FStringLiteralElement - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__function_arguments.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__function_arguments.snap similarity index 80% rename from crates/ruff_python_ast/tests/snapshots/visitor__function_arguments.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__function_arguments.snap index fa6ebc6ee4..a526cdd144 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__function_arguments.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__function_arguments.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtFunctionDef @@ -16,4 +16,3 @@ expression: trace - Parameter - Parameter - StmtPass - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__function_positional_only_with_default.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__function_positional_only_with_default.snap similarity index 73% rename from crates/ruff_python_ast/tests/snapshots/visitor__function_positional_only_with_default.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__function_positional_only_with_default.snap index a92798a086..c7501b8ae8 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__function_positional_only_with_default.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__function_positional_only_with_default.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtFunctionDef @@ -11,4 +11,3 @@ expression: trace - Parameter - Parameter - StmtPass - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__function_type_parameters.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__function_type_parameters.snap similarity index 75% rename from crates/ruff_python_ast/tests/snapshots/visitor__function_type_parameters.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__function_type_parameters.snap index 60a1abe6be..2af2fa85aa 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__function_type_parameters.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__function_type_parameters.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtFunctionDef @@ -11,4 +11,3 @@ expression: trace - Parameters - StmtExpr - ExprEllipsisLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__list_comprehension.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__list_comprehension.snap similarity index 65% rename from crates/ruff_python_ast/tests/snapshots/visitor__list_comprehension.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__list_comprehension.snap index 05f75bd8df..7af26ca3c4 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__list_comprehension.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__list_comprehension.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtExpr @@ -8,4 +8,3 @@ expression: trace - ExprName - ExprName - ExprName - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__match_class_pattern.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__match_class_pattern.snap similarity index 88% rename from crates/ruff_python_ast/tests/snapshots/visitor__match_class_pattern.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__match_class_pattern.snap index 0c3513b0a6..d91b865a4f 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__match_class_pattern.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__match_class_pattern.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtMatch @@ -24,4 +24,3 @@ expression: trace - ExprNumberLiteral - StmtExpr - ExprEllipsisLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__set_comprehension.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__set_comprehension.snap similarity index 64% rename from crates/ruff_python_ast/tests/snapshots/visitor__set_comprehension.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__set_comprehension.snap index 337ef4624f..799b67aefa 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__set_comprehension.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__set_comprehension.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtExpr @@ -8,4 +8,3 @@ expression: trace - ExprName - ExprName - ExprName - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__string_literals.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__string_literals.snap similarity index 64% rename from crates/ruff_python_ast/tests/snapshots/visitor__string_literals.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__string_literals.snap index 7c066eae56..08cd420a99 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__string_literals.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__string_literals.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtExpr @@ -7,4 +7,3 @@ expression: trace - StringLiteral - StringLiteral - StringLiteral - diff --git a/crates/ruff_python_ast/tests/snapshots/visitor__type_aliases.snap b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__type_aliases.snap similarity index 76% rename from crates/ruff_python_ast/tests/snapshots/visitor__type_aliases.snap rename to crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__type_aliases.snap index 10b624fbb0..539a38562d 100644 --- a/crates/ruff_python_ast/tests/snapshots/visitor__type_aliases.snap +++ b/crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__type_aliases.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_ast/tests/visitor.rs +source: crates/ruff_python_ast_integration_tests/tests/visitor.rs expression: trace --- - StmtTypeAlias @@ -12,4 +12,3 @@ expression: trace - TypeParamTypeVarTuple - TypeParamParamSpec - ExprName - diff --git a/crates/ruff_python_ast/tests/stmt_if.rs b/crates/ruff_python_ast_integration_tests/tests/stmt_if.rs similarity index 99% rename from crates/ruff_python_ast/tests/stmt_if.rs rename to crates/ruff_python_ast_integration_tests/tests/stmt_if.rs index fe6c4a85c1..cacf964996 100644 --- a/crates/ruff_python_ast/tests/stmt_if.rs +++ b/crates/ruff_python_ast_integration_tests/tests/stmt_if.rs @@ -1,5 +1,4 @@ use ruff_python_ast::stmt_if::elif_else_range; - use ruff_python_parser::{parse_suite, ParseError}; use ruff_text_size::TextSize; diff --git a/crates/ruff_python_ast/tests/visitor.rs b/crates/ruff_python_ast_integration_tests/tests/visitor.rs similarity index 96% rename from crates/ruff_python_ast/tests/visitor.rs rename to crates/ruff_python_ast_integration_tests/tests/visitor.rs index e039e78d96..1c1bf0d0f7 100644 --- a/crates/ruff_python_ast/tests/visitor.rs +++ b/crates/ruff_python_ast_integration_tests/tests/visitor.rs @@ -1,9 +1,6 @@ use std::fmt::{Debug, Write}; use insta::assert_snapshot; -use ruff_python_ast as ast; -use ruff_python_parser::lexer::lex; -use ruff_python_parser::{parse_tokens, Mode}; use ruff_python_ast::visitor::{ walk_alias, walk_bytes_literal, walk_comprehension, walk_except_handler, walk_expr, @@ -11,12 +8,13 @@ use ruff_python_ast::visitor::{ walk_parameters, walk_pattern, walk_stmt, walk_string_literal, walk_type_param, walk_with_item, Visitor, }; -use ruff_python_ast::AnyNodeRef; use ruff_python_ast::{ - Alias, BoolOp, BytesLiteral, CmpOp, Comprehension, ExceptHandler, Expr, FString, - FStringElement, Keyword, MatchCase, Operator, Parameter, Parameters, Pattern, Stmt, - StringLiteral, TypeParam, UnaryOp, WithItem, + self as ast, Alias, AnyNodeRef, BoolOp, BytesLiteral, CmpOp, Comprehension, ExceptHandler, + Expr, FString, FStringElement, Keyword, MatchCase, Operator, Parameter, Parameters, Pattern, + Stmt, StringLiteral, TypeParam, UnaryOp, WithItem, }; +use ruff_python_parser::lexer::lex; +use ruff_python_parser::{parse_tokens, Mode}; #[test] fn function_arguments() { diff --git a/crates/ruff_python_trivia/Cargo.toml b/crates/ruff_python_trivia/Cargo.toml index 9e92e1eb52..98d5b60220 100644 --- a/crates/ruff_python_trivia/Cargo.toml +++ b/crates/ruff_python_trivia/Cargo.toml @@ -21,8 +21,6 @@ unicode-ident = { workspace = true } [dev-dependencies] insta = { workspace = true } -ruff_python_parser = { path = "../ruff_python_parser" } -ruff_python_index = { path = "../ruff_python_index" } [lints] workspace = true diff --git a/crates/ruff_python_trivia/src/comment_ranges.rs b/crates/ruff_python_trivia/src/comment_ranges.rs index bdec7c428e..2c5c060657 100644 --- a/crates/ruff_python_trivia/src/comment_ranges.rs +++ b/crates/ruff_python_trivia/src/comment_ranges.rs @@ -203,158 +203,3 @@ impl<'a> IntoIterator for &'a CommentRanges { self.raw.iter() } } - -#[cfg(test)] -mod tests { - use ruff_python_index::Indexer; - use ruff_python_parser::lexer::LexResult; - use ruff_python_parser::{tokenize, Mode}; - use ruff_source_file::Locator; - use ruff_text_size::TextSize; - - #[test] - fn block_comments_two_line_block_at_start() { - // arrange - let source = "# line 1\n# line 2\n"; - let tokens = tokenize(source, Mode::Module); - let locator = Locator::new(source); - let indexer = Indexer::from_tokens(&tokens, &locator); - - // act - let block_comments = indexer.comment_ranges().block_comments(&locator); - - // assert - assert_eq!(block_comments, vec![TextSize::new(0), TextSize::new(9)]); - } - - #[test] - fn block_comments_indented_block() { - // arrange - let source = " # line 1\n # line 2\n"; - let tokens = tokenize(source, Mode::Module); - let locator = Locator::new(source); - let indexer = Indexer::from_tokens(&tokens, &locator); - - // act - let block_comments = indexer.comment_ranges().block_comments(&locator); - - // assert - assert_eq!(block_comments, vec![TextSize::new(4), TextSize::new(17)]); - } - - #[test] - fn block_comments_single_line_is_not_a_block() { - // arrange - let source = "\n"; - let tokens: Vec = tokenize(source, Mode::Module); - let locator = Locator::new(source); - let indexer = Indexer::from_tokens(&tokens, &locator); - - // act - let block_comments = indexer.comment_ranges().block_comments(&locator); - - // assert - assert_eq!(block_comments, Vec::::new()); - } - - #[test] - fn block_comments_lines_with_code_not_a_block() { - // arrange - let source = "x = 1 # line 1\ny = 2 # line 2\n"; - let tokens = tokenize(source, Mode::Module); - let locator = Locator::new(source); - let indexer = Indexer::from_tokens(&tokens, &locator); - - // act - let block_comments = indexer.comment_ranges().block_comments(&locator); - - // assert - assert_eq!(block_comments, Vec::::new()); - } - - #[test] - fn block_comments_sequential_lines_not_in_block() { - // arrange - let source = " # line 1\n # line 2\n"; - let tokens = tokenize(source, Mode::Module); - let locator = Locator::new(source); - let indexer = Indexer::from_tokens(&tokens, &locator); - - // act - let block_comments = indexer.comment_ranges().block_comments(&locator); - - // assert - assert_eq!(block_comments, Vec::::new()); - } - - #[test] - fn block_comments_lines_in_triple_quotes_not_a_block() { - // arrange - let source = r#" - """ - # line 1 - # line 2 - """ - "#; - let tokens = tokenize(source, Mode::Module); - let locator = Locator::new(source); - let indexer = Indexer::from_tokens(&tokens, &locator); - - // act - let block_comments = indexer.comment_ranges().block_comments(&locator); - - // assert - assert_eq!(block_comments, Vec::::new()); - } - - #[test] - fn block_comments_stress_test() { - // arrange - let source = r#" -# block comment 1 line 1 -# block comment 2 line 2 - -# these lines - # do not form -# a block comment - -x = 1 # these lines also do not -y = 2 # do not form a block comment - -# these lines do form a block comment -# - - # - # and so do these - # - -""" -# these lines are in triple quotes and -# therefore do not form a block comment -""" - "#; - let tokens = tokenize(source, Mode::Module); - let locator = Locator::new(source); - let indexer = Indexer::from_tokens(&tokens, &locator); - - // act - let block_comments = indexer.comment_ranges().block_comments(&locator); - - // assert - assert_eq!( - block_comments, - vec![ - // Block #1 - TextSize::new(1), - TextSize::new(26), - // Block #2 - TextSize::new(174), - TextSize::new(212), - // Block #3 - TextSize::new(219), - TextSize::new(225), - TextSize::new(247) - ] - ); - } -} diff --git a/crates/ruff_python_trivia/src/tokenizer.rs b/crates/ruff_python_trivia/src/tokenizer.rs index d1135bb9bc..13181a7489 100644 --- a/crates/ruff_python_trivia/src/tokenizer.rs +++ b/crates/ruff_python_trivia/src/tokenizer.rs @@ -1024,426 +1024,3 @@ impl Iterator for BackwardsTokenizer<'_> { } } } - -#[cfg(test)] -mod tests { - use insta::assert_debug_snapshot; - - use ruff_python_parser::lexer::lex; - use ruff_python_parser::{Mode, Tok}; - use ruff_text_size::{TextLen, TextRange, TextSize}; - - use crate::tokenizer::{lines_after, lines_before, SimpleToken, SimpleTokenizer}; - use crate::{BackwardsTokenizer, SimpleTokenKind}; - - struct TokenizationTestCase { - source: &'static str, - range: TextRange, - tokens: Vec, - } - - impl TokenizationTestCase { - fn assert_reverse_tokenization(&self) { - let mut backwards = self.tokenize_reverse(); - - // Re-reverse to get the tokens in forward order. - backwards.reverse(); - - assert_eq!(&backwards, &self.tokens); - } - - fn tokenize_reverse(&self) -> Vec { - let comment_ranges: Vec<_> = lex(self.source, Mode::Module) - .filter_map(|result| { - let (token, range) = result.expect("Input to be a valid python program."); - if matches!(token, Tok::Comment(_)) { - Some(range) - } else { - None - } - }) - .collect(); - BackwardsTokenizer::new(self.source, self.range, &comment_ranges).collect() - } - - fn tokens(&self) -> &[SimpleToken] { - &self.tokens - } - } - - fn tokenize_range(source: &'static str, range: TextRange) -> TokenizationTestCase { - let tokens: Vec<_> = SimpleTokenizer::new(source, range).collect(); - - TokenizationTestCase { - source, - range, - tokens, - } - } - - fn tokenize(source: &'static str) -> TokenizationTestCase { - tokenize_range(source, TextRange::new(TextSize::new(0), source.text_len())) - } - - #[test] - fn tokenize_trivia() { - let source = "# comment\n # comment"; - - let test_case = tokenize(source); - - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn tokenize_parentheses() { - let source = "([{}])"; - - let test_case = tokenize(source); - - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn tokenize_comma() { - let source = ",,,,"; - - let test_case = tokenize(source); - - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn tokenize_eq() { - // Should tokenize as `==`, then `=`, regardless of whether we're lexing forwards or - // backwards. - let source = "==="; - - let test_case = tokenize(source); - - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn tokenize_not_eq() { - // Should tokenize as `!=`, then `=`, regardless of whether we're lexing forwards or - // backwards. - let source = "!=="; - - let test_case = tokenize(source); - - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn tokenize_continuation() { - let source = "( \\\n )"; - - let test_case = tokenize(source); - - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn tokenize_operators() { - let source = "-> *= ( -= ) ~ // ** **= ^ ^= | |="; - - let test_case = tokenize(source); - - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn tokenize_invalid_operators() { - let source = "-> $="; - - let test_case = tokenize(source); - - assert_debug_snapshot!(test_case.tokens()); - - // note: not reversible: [other, bogus, bogus] vs [bogus, bogus, other] - } - - #[test] - fn tricky_unicode() { - let source = "មុ"; - - let test_case = tokenize(source); - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn identifier_ending_in_non_start_char() { - let source = "i5"; - - let test_case = tokenize(source); - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn string_with_kind() { - let source = "f'foo'"; - - let test_case = tokenize(source); - assert_debug_snapshot!(test_case.tokens()); - - // note: not reversible: [other, bogus] vs [bogus, other] - } - - #[test] - fn string_with_byte_kind() { - let source = "BR'foo'"; - - let test_case = tokenize(source); - assert_debug_snapshot!(test_case.tokens()); - - // note: not reversible: [other, bogus] vs [bogus, other] - } - - #[test] - fn string_with_invalid_kind() { - let source = "abc'foo'"; - - let test_case = tokenize(source); - assert_debug_snapshot!(test_case.tokens()); - - // note: not reversible: [other, bogus] vs [bogus, other] - } - - #[test] - fn identifier_starting_with_string_kind() { - let source = "foo bar"; - - let test_case = tokenize(source); - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn ignore_word_with_only_id_continuing_chars() { - let source = "555"; - - let test_case = tokenize(source); - assert_debug_snapshot!(test_case.tokens()); - - // note: not reversible: [other, bogus, bogus] vs [bogus, bogus, other] - } - - #[test] - fn tokenize_multichar() { - let source = "if in else match"; - - let test_case = tokenize(source); - - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn tokenize_substring() { - let source = "('some string') # comment"; - - let test_case = - tokenize_range(source, TextRange::new(TextSize::new(14), source.text_len())); - - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn tokenize_slash() { - let source = r" # trailing positional comment - # Positional arguments only after here - ,/"; - - let test_case = tokenize(source); - - assert_debug_snapshot!(test_case.tokens()); - test_case.assert_reverse_tokenization(); - } - - #[test] - fn tokenize_bogus() { - let source = r#"# leading comment - "a string" - a = (10)"#; - - let test_case = tokenize(source); - - assert_debug_snapshot!(test_case.tokens()); - assert_debug_snapshot!("Reverse", test_case.tokenize_reverse()); - } - - #[test] - fn single_quoted_multiline_string_containing_comment() { - let test_case = tokenize( - r"'This string contains a hash looking like a comment\ -# This is not a comment'", - ); - - assert_debug_snapshot!(test_case.tokenize_reverse()); - } - - #[test] - fn single_quoted_multiline_string_implicit_concatenation() { - let test_case = tokenize( - r#"'This string contains a hash looking like a comment\ -# This is' "not_a_comment""#, - ); - - assert_debug_snapshot!(test_case.tokenize_reverse()); - } - - #[test] - fn triple_quoted_multiline_string_containing_comment() { - let test_case = tokenize( - r"'''This string contains a hash looking like a comment -# This is not a comment'''", - ); - - assert_debug_snapshot!(test_case.tokenize_reverse()); - } - - #[test] - fn comment_containing_triple_quoted_string() { - let test_case = tokenize("'''leading string''' # a comment '''not a string'''"); - - assert_debug_snapshot!(test_case.tokenize_reverse()); - } - - #[test] - fn comment_containing_single_quoted_string() { - let test_case = tokenize("'leading string' # a comment 'not a string'"); - - assert_debug_snapshot!(test_case.tokenize_reverse()); - } - - #[test] - fn string_followed_by_multiple_comments() { - let test_case = - tokenize(r#"'a string # containing a hash " # and another hash ' # finally a comment"#); - - assert_debug_snapshot!(test_case.tokenize_reverse()); - } - - #[test] - fn string_with_escaped_quote() { - let test_case = tokenize(r"'a string \' # containing a hash ' # finally a comment"); - - assert_debug_snapshot!(test_case.tokenize_reverse()); - } - - #[test] - fn string_with_double_escaped_backslash() { - let test_case = tokenize(r"'a string \\' # a comment '"); - - assert_debug_snapshot!(test_case.tokenize_reverse()); - } - - #[test] - fn empty_string_literal() { - let test_case = tokenize(r"'' # a comment '"); - - assert_debug_snapshot!(test_case.tokenize_reverse()); - } - - #[test] - fn lines_before_empty_string() { - assert_eq!(lines_before(TextSize::new(0), ""), 0); - } - - #[test] - fn lines_before_in_the_middle_of_a_line() { - assert_eq!(lines_before(TextSize::new(4), "a = 20"), 0); - } - - #[test] - fn lines_before_on_a_new_line() { - assert_eq!(lines_before(TextSize::new(7), "a = 20\nb = 10"), 1); - } - - #[test] - fn lines_before_multiple_leading_newlines() { - assert_eq!(lines_before(TextSize::new(9), "a = 20\n\r\nb = 10"), 2); - } - - #[test] - fn lines_before_with_comment_offset() { - assert_eq!(lines_before(TextSize::new(8), "a = 20\n# a comment"), 0); - } - - #[test] - fn lines_before_with_trailing_comment() { - assert_eq!( - lines_before(TextSize::new(22), "a = 20 # some comment\nb = 10"), - 1 - ); - } - - #[test] - fn lines_before_with_comment_only_line() { - assert_eq!( - lines_before(TextSize::new(22), "a = 20\n# some comment\nb = 10"), - 1 - ); - } - - #[test] - fn lines_after_empty_string() { - assert_eq!(lines_after(TextSize::new(0), ""), 0); - } - - #[test] - fn lines_after_in_the_middle_of_a_line() { - assert_eq!(lines_after(TextSize::new(4), "a = 20"), 0); - } - - #[test] - fn lines_after_before_a_new_line() { - assert_eq!(lines_after(TextSize::new(6), "a = 20\nb = 10"), 1); - } - - #[test] - fn lines_after_multiple_newlines() { - assert_eq!(lines_after(TextSize::new(6), "a = 20\n\r\nb = 10"), 2); - } - - #[test] - fn lines_after_before_comment_offset() { - assert_eq!(lines_after(TextSize::new(7), "a = 20 # a comment\n"), 0); - } - - #[test] - fn lines_after_with_comment_only_line() { - assert_eq!( - lines_after(TextSize::new(6), "a = 20\n# some comment\nb = 10"), - 1 - ); - } - - #[test] - fn test_previous_token_simple() { - let cases = &["x = (", "x = ( ", "x = (\n"]; - for source in cases { - let token = BackwardsTokenizer::up_to(source.text_len(), source, &[]) - .skip_trivia() - .next() - .unwrap(); - assert_eq!( - token, - SimpleToken { - kind: SimpleTokenKind::LParen, - range: TextRange::new(TextSize::new(4), TextSize::new(5)), - } - ); - } - } -} diff --git a/crates/ruff_python_trivia/src/whitespace.rs b/crates/ruff_python_trivia/src/whitespace.rs index 9e91cfb69e..7b8b5e90ea 100644 --- a/crates/ruff_python_trivia/src/whitespace.rs +++ b/crates/ruff_python_trivia/src/whitespace.rs @@ -79,51 +79,3 @@ impl PythonWhitespace for str { self.trim_end_matches(is_python_whitespace) } } - -#[cfg(test)] -mod tests { - use ruff_python_parser::{parse_suite, ParseError}; - use ruff_source_file::Locator; - use ruff_text_size::Ranged; - - use crate::has_trailing_content; - - #[test] - fn trailing_content() -> Result<(), ParseError> { - let contents = "x = 1"; - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); - let locator = Locator::new(contents); - assert!(!has_trailing_content(stmt.end(), &locator)); - - let contents = "x = 1; y = 2"; - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); - let locator = Locator::new(contents); - assert!(has_trailing_content(stmt.end(), &locator)); - - let contents = "x = 1 "; - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); - let locator = Locator::new(contents); - assert!(!has_trailing_content(stmt.end(), &locator)); - - let contents = "x = 1 # Comment"; - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); - let locator = Locator::new(contents); - assert!(!has_trailing_content(stmt.end(), &locator)); - - let contents = r" -x = 1 -y = 2 -" - .trim(); - let program = parse_suite(contents)?; - let stmt = program.first().unwrap(); - let locator = Locator::new(contents); - assert!(!has_trailing_content(stmt.end(), &locator)); - - Ok(()) - } -} diff --git a/crates/ruff_python_trivia_integration_tests/Cargo.toml b/crates/ruff_python_trivia_integration_tests/Cargo.toml new file mode 100644 index 0000000000..cb144472fb --- /dev/null +++ b/crates/ruff_python_trivia_integration_tests/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "ruff_python_trivia_integration_tests" +version = "0.0.0" +edition.workspace = true +rust-version.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] + +[dev-dependencies] +ruff_python_index = { path = "../ruff_python_index" } +ruff_python_parser = { path = "../ruff_python_parser" } +ruff_python_trivia = { path = "../ruff_python_trivia" } +ruff_source_file = { path = "../ruff_source_file" } +ruff_text_size = { path = "../ruff_text_size" } + +insta = { workspace = true } + +[lints] +workspace = true diff --git a/crates/ruff_python_trivia_integration_tests/README.md b/crates/ruff_python_trivia_integration_tests/README.md new file mode 100644 index 0000000000..78024349ea --- /dev/null +++ b/crates/ruff_python_trivia_integration_tests/README.md @@ -0,0 +1,13 @@ +# Integration tests for `ruff_python_trivia` + +This crate includes integration tests for the `ruff_python_trivia` crate. + +The reason for having a separate crate is to avoid introducing a dev circular +dependency between the `ruff_python_parser` crate and the `ruff_python_trivia` crate. + +This crate shouldn't include any code, only tests. + +**Reference:** + +- `rust-analyzer` issue: +- Ruff's pull request: diff --git a/crates/ruff_python_trivia_integration_tests/src/lib.rs b/crates/ruff_python_trivia_integration_tests/src/lib.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crates/ruff_python_trivia_integration_tests/src/lib.rs @@ -0,0 +1 @@ + diff --git a/crates/ruff_python_trivia_integration_tests/tests/block_comments.rs b/crates/ruff_python_trivia_integration_tests/tests/block_comments.rs new file mode 100644 index 0000000000..df0142b3c1 --- /dev/null +++ b/crates/ruff_python_trivia_integration_tests/tests/block_comments.rs @@ -0,0 +1,151 @@ +use ruff_python_index::Indexer; +use ruff_python_parser::lexer::LexResult; +use ruff_python_parser::{tokenize, Mode}; +use ruff_source_file::Locator; +use ruff_text_size::TextSize; + +#[test] +fn block_comments_two_line_block_at_start() { + // arrange + let source = "# line 1\n# line 2\n"; + let tokens = tokenize(source, Mode::Module); + let locator = Locator::new(source); + let indexer = Indexer::from_tokens(&tokens, &locator); + + // act + let block_comments = indexer.comment_ranges().block_comments(&locator); + + // assert + assert_eq!(block_comments, vec![TextSize::new(0), TextSize::new(9)]); +} + +#[test] +fn block_comments_indented_block() { + // arrange + let source = " # line 1\n # line 2\n"; + let tokens = tokenize(source, Mode::Module); + let locator = Locator::new(source); + let indexer = Indexer::from_tokens(&tokens, &locator); + + // act + let block_comments = indexer.comment_ranges().block_comments(&locator); + + // assert + assert_eq!(block_comments, vec![TextSize::new(4), TextSize::new(17)]); +} + +#[test] +fn block_comments_single_line_is_not_a_block() { + // arrange + let source = "\n"; + let tokens: Vec = tokenize(source, Mode::Module); + let locator = Locator::new(source); + let indexer = Indexer::from_tokens(&tokens, &locator); + + // act + let block_comments = indexer.comment_ranges().block_comments(&locator); + + // assert + assert_eq!(block_comments, Vec::::new()); +} + +#[test] +fn block_comments_lines_with_code_not_a_block() { + // arrange + let source = "x = 1 # line 1\ny = 2 # line 2\n"; + let tokens = tokenize(source, Mode::Module); + let locator = Locator::new(source); + let indexer = Indexer::from_tokens(&tokens, &locator); + + // act + let block_comments = indexer.comment_ranges().block_comments(&locator); + + // assert + assert_eq!(block_comments, Vec::::new()); +} + +#[test] +fn block_comments_sequential_lines_not_in_block() { + // arrange + let source = " # line 1\n # line 2\n"; + let tokens = tokenize(source, Mode::Module); + let locator = Locator::new(source); + let indexer = Indexer::from_tokens(&tokens, &locator); + + // act + let block_comments = indexer.comment_ranges().block_comments(&locator); + + // assert + assert_eq!(block_comments, Vec::::new()); +} + +#[test] +fn block_comments_lines_in_triple_quotes_not_a_block() { + // arrange + let source = r#" + """ + # line 1 + # line 2 + """ + "#; + let tokens = tokenize(source, Mode::Module); + let locator = Locator::new(source); + let indexer = Indexer::from_tokens(&tokens, &locator); + + // act + let block_comments = indexer.comment_ranges().block_comments(&locator); + + // assert + assert_eq!(block_comments, Vec::::new()); +} + +#[test] +fn block_comments_stress_test() { + // arrange + let source = r#" +# block comment 1 line 1 +# block comment 2 line 2 + +# these lines + # do not form +# a block comment + +x = 1 # these lines also do not +y = 2 # do not form a block comment + +# these lines do form a block comment +# + + # + # and so do these + # + +""" +# these lines are in triple quotes and +# therefore do not form a block comment +""" + "#; + let tokens = tokenize(source, Mode::Module); + let locator = Locator::new(source); + let indexer = Indexer::from_tokens(&tokens, &locator); + + // act + let block_comments = indexer.comment_ranges().block_comments(&locator); + + // assert + assert_eq!( + block_comments, + vec![ + // Block #1 + TextSize::new(1), + TextSize::new(26), + // Block #2 + TextSize::new(174), + TextSize::new(212), + // Block #3 + TextSize::new(219), + TextSize::new(225), + TextSize::new(247) + ] + ); +} diff --git a/crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs b/crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs new file mode 100644 index 0000000000..5ac4296ea6 --- /dev/null +++ b/crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs @@ -0,0 +1,417 @@ +use insta::assert_debug_snapshot; + +use ruff_python_parser::lexer::lex; +use ruff_python_parser::{Mode, Tok}; +use ruff_python_trivia::{lines_after, lines_before, SimpleToken, SimpleTokenizer}; +use ruff_python_trivia::{BackwardsTokenizer, SimpleTokenKind}; +use ruff_text_size::{TextLen, TextRange, TextSize}; + +struct TokenizationTestCase { + source: &'static str, + range: TextRange, + tokens: Vec, +} + +impl TokenizationTestCase { + fn assert_reverse_tokenization(&self) { + let mut backwards = self.tokenize_reverse(); + + // Re-reverse to get the tokens in forward order. + backwards.reverse(); + + assert_eq!(&backwards, &self.tokens); + } + + fn tokenize_reverse(&self) -> Vec { + let comment_ranges: Vec<_> = lex(self.source, Mode::Module) + .filter_map(|result| { + let (token, range) = result.expect("Input to be a valid python program."); + if matches!(token, Tok::Comment(_)) { + Some(range) + } else { + None + } + }) + .collect(); + BackwardsTokenizer::new(self.source, self.range, &comment_ranges).collect() + } + + fn tokens(&self) -> &[SimpleToken] { + &self.tokens + } +} + +fn tokenize_range(source: &'static str, range: TextRange) -> TokenizationTestCase { + let tokens: Vec<_> = SimpleTokenizer::new(source, range).collect(); + + TokenizationTestCase { + source, + range, + tokens, + } +} + +fn tokenize(source: &'static str) -> TokenizationTestCase { + tokenize_range(source, TextRange::new(TextSize::new(0), source.text_len())) +} + +#[test] +fn tokenize_trivia() { + let source = "# comment\n # comment"; + + let test_case = tokenize(source); + + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn tokenize_parentheses() { + let source = "([{}])"; + + let test_case = tokenize(source); + + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn tokenize_comma() { + let source = ",,,,"; + + let test_case = tokenize(source); + + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn tokenize_eq() { + // Should tokenize as `==`, then `=`, regardless of whether we're lexing forwards or + // backwards. + let source = "==="; + + let test_case = tokenize(source); + + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn tokenize_not_eq() { + // Should tokenize as `!=`, then `=`, regardless of whether we're lexing forwards or + // backwards. + let source = "!=="; + + let test_case = tokenize(source); + + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn tokenize_continuation() { + let source = "( \\\n )"; + + let test_case = tokenize(source); + + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn tokenize_operators() { + let source = "-> *= ( -= ) ~ // ** **= ^ ^= | |="; + + let test_case = tokenize(source); + + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn tokenize_invalid_operators() { + let source = "-> $="; + + let test_case = tokenize(source); + + assert_debug_snapshot!(test_case.tokens()); + + // note: not reversible: [other, bogus, bogus] vs [bogus, bogus, other] +} + +#[test] +fn tricky_unicode() { + let source = "មុ"; + + let test_case = tokenize(source); + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn identifier_ending_in_non_start_char() { + let source = "i5"; + + let test_case = tokenize(source); + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn string_with_kind() { + let source = "f'foo'"; + + let test_case = tokenize(source); + assert_debug_snapshot!(test_case.tokens()); + + // note: not reversible: [other, bogus] vs [bogus, other] +} + +#[test] +fn string_with_byte_kind() { + let source = "BR'foo'"; + + let test_case = tokenize(source); + assert_debug_snapshot!(test_case.tokens()); + + // note: not reversible: [other, bogus] vs [bogus, other] +} + +#[test] +fn string_with_invalid_kind() { + let source = "abc'foo'"; + + let test_case = tokenize(source); + assert_debug_snapshot!(test_case.tokens()); + + // note: not reversible: [other, bogus] vs [bogus, other] +} + +#[test] +fn identifier_starting_with_string_kind() { + let source = "foo bar"; + + let test_case = tokenize(source); + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn ignore_word_with_only_id_continuing_chars() { + let source = "555"; + + let test_case = tokenize(source); + assert_debug_snapshot!(test_case.tokens()); + + // note: not reversible: [other, bogus, bogus] vs [bogus, bogus, other] +} + +#[test] +fn tokenize_multichar() { + let source = "if in else match"; + + let test_case = tokenize(source); + + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn tokenize_substring() { + let source = "('some string') # comment"; + + let test_case = tokenize_range(source, TextRange::new(TextSize::new(14), source.text_len())); + + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn tokenize_slash() { + let source = r" # trailing positional comment + # Positional arguments only after here + ,/"; + + let test_case = tokenize(source); + + assert_debug_snapshot!(test_case.tokens()); + test_case.assert_reverse_tokenization(); +} + +#[test] +fn tokenize_bogus() { + let source = r#"# leading comment + "a string" + a = (10)"#; + + let test_case = tokenize(source); + + assert_debug_snapshot!(test_case.tokens()); + assert_debug_snapshot!("Reverse", test_case.tokenize_reverse()); +} + +#[test] +fn single_quoted_multiline_string_containing_comment() { + let test_case = tokenize( + r"'This string contains a hash looking like a comment\ +# This is not a comment'", + ); + + assert_debug_snapshot!(test_case.tokenize_reverse()); +} + +#[test] +fn single_quoted_multiline_string_implicit_concatenation() { + let test_case = tokenize( + r#"'This string contains a hash looking like a comment\ +# This is' "not_a_comment""#, + ); + + assert_debug_snapshot!(test_case.tokenize_reverse()); +} + +#[test] +fn triple_quoted_multiline_string_containing_comment() { + let test_case = tokenize( + r"'''This string contains a hash looking like a comment +# This is not a comment'''", + ); + + assert_debug_snapshot!(test_case.tokenize_reverse()); +} + +#[test] +fn comment_containing_triple_quoted_string() { + let test_case = tokenize("'''leading string''' # a comment '''not a string'''"); + + assert_debug_snapshot!(test_case.tokenize_reverse()); +} + +#[test] +fn comment_containing_single_quoted_string() { + let test_case = tokenize("'leading string' # a comment 'not a string'"); + + assert_debug_snapshot!(test_case.tokenize_reverse()); +} + +#[test] +fn string_followed_by_multiple_comments() { + let test_case = + tokenize(r#"'a string # containing a hash " # and another hash ' # finally a comment"#); + + assert_debug_snapshot!(test_case.tokenize_reverse()); +} + +#[test] +fn string_with_escaped_quote() { + let test_case = tokenize(r"'a string \' # containing a hash ' # finally a comment"); + + assert_debug_snapshot!(test_case.tokenize_reverse()); +} + +#[test] +fn string_with_double_escaped_backslash() { + let test_case = tokenize(r"'a string \\' # a comment '"); + + assert_debug_snapshot!(test_case.tokenize_reverse()); +} + +#[test] +fn empty_string_literal() { + let test_case = tokenize(r"'' # a comment '"); + + assert_debug_snapshot!(test_case.tokenize_reverse()); +} + +#[test] +fn lines_before_empty_string() { + assert_eq!(lines_before(TextSize::new(0), ""), 0); +} + +#[test] +fn lines_before_in_the_middle_of_a_line() { + assert_eq!(lines_before(TextSize::new(4), "a = 20"), 0); +} + +#[test] +fn lines_before_on_a_new_line() { + assert_eq!(lines_before(TextSize::new(7), "a = 20\nb = 10"), 1); +} + +#[test] +fn lines_before_multiple_leading_newlines() { + assert_eq!(lines_before(TextSize::new(9), "a = 20\n\r\nb = 10"), 2); +} + +#[test] +fn lines_before_with_comment_offset() { + assert_eq!(lines_before(TextSize::new(8), "a = 20\n# a comment"), 0); +} + +#[test] +fn lines_before_with_trailing_comment() { + assert_eq!( + lines_before(TextSize::new(22), "a = 20 # some comment\nb = 10"), + 1 + ); +} + +#[test] +fn lines_before_with_comment_only_line() { + assert_eq!( + lines_before(TextSize::new(22), "a = 20\n# some comment\nb = 10"), + 1 + ); +} + +#[test] +fn lines_after_empty_string() { + assert_eq!(lines_after(TextSize::new(0), ""), 0); +} + +#[test] +fn lines_after_in_the_middle_of_a_line() { + assert_eq!(lines_after(TextSize::new(4), "a = 20"), 0); +} + +#[test] +fn lines_after_before_a_new_line() { + assert_eq!(lines_after(TextSize::new(6), "a = 20\nb = 10"), 1); +} + +#[test] +fn lines_after_multiple_newlines() { + assert_eq!(lines_after(TextSize::new(6), "a = 20\n\r\nb = 10"), 2); +} + +#[test] +fn lines_after_before_comment_offset() { + assert_eq!(lines_after(TextSize::new(7), "a = 20 # a comment\n"), 0); +} + +#[test] +fn lines_after_with_comment_only_line() { + assert_eq!( + lines_after(TextSize::new(6), "a = 20\n# some comment\nb = 10"), + 1 + ); +} + +#[test] +fn test_previous_token_simple() { + let cases = &["x = (", "x = ( ", "x = (\n"]; + for source in cases { + let token = BackwardsTokenizer::up_to(source.text_len(), source, &[]) + .skip_trivia() + .next() + .unwrap(); + assert_eq!( + token, + SimpleToken { + kind: SimpleTokenKind::LParen, + range: TextRange::new(TextSize::new(4), TextSize::new(5)), + } + ); + } +} diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__Reverse.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__Reverse.snap similarity index 76% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__Reverse.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__Reverse.snap index 98b020e6e8..7763ffb690 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__Reverse.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__Reverse.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokenize_reverse() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_single_quoted_string.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__comment_containing_single_quoted_string.snap similarity index 81% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_single_quoted_string.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__comment_containing_single_quoted_string.snap index 1ebd8a5f48..36ea89df61 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_single_quoted_string.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__comment_containing_single_quoted_string.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokenize_reverse() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_triple_quoted_string.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__comment_containing_triple_quoted_string.snap similarity index 81% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_triple_quoted_string.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__comment_containing_triple_quoted_string.snap index f2958e6691..a794749cff 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_triple_quoted_string.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__comment_containing_triple_quoted_string.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokenize_reverse() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__empty_string_literal.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__empty_string_literal.snap similarity index 80% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__empty_string_literal.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__empty_string_literal.snap index 7f3571ef27..1d420e049f 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__empty_string_literal.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__empty_string_literal.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokenize_reverse() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__identifier_ending_in_non_start_char.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__identifier_ending_in_non_start_char.snap similarity index 58% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__identifier_ending_in_non_start_char.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__identifier_ending_in_non_start_char.snap index 7b3494de28..32e4f33792 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__identifier_ending_in_non_start_char.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__identifier_ending_in_non_start_char.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__identifier_starting_with_string_kind.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__identifier_starting_with_string_kind.snap similarity index 76% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__identifier_starting_with_string_kind.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__identifier_starting_with_string_kind.snap index 6d8bfb8f06..04086f5524 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__identifier_starting_with_string_kind.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__identifier_starting_with_string_kind.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__ignore_word_with_only_id_continuing_chars.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__ignore_word_with_only_id_continuing_chars.snap similarity index 69% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__ignore_word_with_only_id_continuing_chars.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__ignore_word_with_only_id_continuing_chars.snap index 24b9b16ccc..a1248f3b41 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__ignore_word_with_only_id_continuing_chars.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__ignore_word_with_only_id_continuing_chars.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_containing_comment.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__single_quoted_multiline_string_containing_comment.snap similarity index 70% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_containing_comment.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__single_quoted_multiline_string_containing_comment.snap index 5836fce453..913b789991 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_containing_comment.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__single_quoted_multiline_string_containing_comment.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokenize_reverse() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_implicit_concatenation.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__single_quoted_multiline_string_implicit_concatenation.snap similarity index 70% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_implicit_concatenation.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__single_quoted_multiline_string_implicit_concatenation.snap index 774475fc23..88771f8ce2 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_implicit_concatenation.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__single_quoted_multiline_string_implicit_concatenation.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokenize_reverse() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_followed_by_multiple_comments.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_followed_by_multiple_comments.snap similarity index 81% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_followed_by_multiple_comments.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_followed_by_multiple_comments.snap index 01da094943..2c623fbaad 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_followed_by_multiple_comments.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_followed_by_multiple_comments.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokenize_reverse() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_byte_kind.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_byte_kind.snap similarity index 69% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_byte_kind.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_byte_kind.snap index 5b535116cb..13e26326e6 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_byte_kind.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_byte_kind.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_double_escaped_backslash.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_double_escaped_backslash.snap similarity index 81% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_double_escaped_backslash.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_double_escaped_backslash.snap index ce46e6154a..84ac21254d 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_double_escaped_backslash.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_double_escaped_backslash.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokenize_reverse() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_escaped_quote.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_escaped_quote.snap similarity index 81% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_escaped_quote.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_escaped_quote.snap index 7e1735c3ed..1bf0706e3a 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_escaped_quote.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_escaped_quote.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokenize_reverse() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_invalid_kind.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_invalid_kind.snap similarity index 75% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_invalid_kind.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_invalid_kind.snap index cab8e69c89..cdcaad2ccb 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_invalid_kind.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_invalid_kind.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_kind.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_kind.snap similarity index 69% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_kind.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_kind.snap index a3030fc9af..4f20942cc6 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_kind.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__string_with_kind.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_bogus.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_bogus.snap similarity index 83% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_bogus.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_bogus.snap index 1b2a383429..ab66aa9c11 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_bogus.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_bogus.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_comma.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_comma.snap similarity index 79% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_comma.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_comma.snap index d8e5794877..be4f7308d7 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_comma.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_comma.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_continuation.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_continuation.snap similarity index 85% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_continuation.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_continuation.snap index 9c4f76b3e7..801a7a65dd 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_continuation.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_continuation.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_eq.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_eq.snap similarity index 69% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_eq.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_eq.snap index 26f9c5ae2c..4bab77b681 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_eq.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_eq.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_invalid_operators.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_invalid_operators.snap similarity index 80% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_invalid_operators.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_invalid_operators.snap index cee59e0ba3..78f0374943 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_invalid_operators.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_invalid_operators.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_multichar.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_multichar.snap similarity index 87% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_multichar.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_multichar.snap index 1f2da93d5d..4cc2111b55 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_multichar.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_multichar.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_not_eq.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_not_eq.snap similarity index 69% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_not_eq.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_not_eq.snap index 00a7d9c1fc..e2db410347 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_not_eq.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_not_eq.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_operators.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_operators.snap similarity index 95% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_operators.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_operators.snap index f0f92f80c0..e0430a9553 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_operators.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_operators.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_parentheses.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_parentheses.snap similarity index 85% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_parentheses.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_parentheses.snap index 7a2fa7db11..0e3bedccd5 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_parentheses.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_parentheses.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_slash.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_slash.snap similarity index 89% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_slash.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_slash.snap index fe0d7b1eaa..43c5d6d815 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_slash.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_slash.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_substring.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_substring.snap similarity index 76% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_substring.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_substring.snap index 66bd085333..d3b74a1074 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_substring.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_substring.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_trivia.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_trivia.snap similarity index 80% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_trivia.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_trivia.snap index 476f426dec..4872572e91 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_trivia.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tokenize_trivia.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tricky_unicode.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tricky_unicode.snap similarity index 58% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tricky_unicode.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tricky_unicode.snap index 81864953b5..bc8a977653 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tricky_unicode.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__tricky_unicode.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokens() --- [ diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__triple_quoted_multiline_string_containing_comment.snap b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__triple_quoted_multiline_string_containing_comment.snap similarity index 70% rename from crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__triple_quoted_multiline_string_containing_comment.snap rename to crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__triple_quoted_multiline_string_containing_comment.snap index 2708eac19e..e2081323a1 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__triple_quoted_multiline_string_containing_comment.snap +++ b/crates/ruff_python_trivia_integration_tests/tests/snapshots/simple_tokenizer__triple_quoted_multiline_string_containing_comment.snap @@ -1,5 +1,5 @@ --- -source: crates/ruff_python_trivia/src/tokenizer.rs +source: crates/ruff_python_trivia_integration_tests/tests/simple_tokenizer.rs expression: test_case.tokenize_reverse() --- [ diff --git a/crates/ruff_python_trivia_integration_tests/tests/whitespace.rs b/crates/ruff_python_trivia_integration_tests/tests/whitespace.rs new file mode 100644 index 0000000000..709a3a3d18 --- /dev/null +++ b/crates/ruff_python_trivia_integration_tests/tests/whitespace.rs @@ -0,0 +1,43 @@ +use ruff_python_parser::{parse_suite, ParseError}; +use ruff_python_trivia::has_trailing_content; +use ruff_source_file::Locator; +use ruff_text_size::Ranged; + +#[test] +fn trailing_content() -> Result<(), ParseError> { + let contents = "x = 1"; + let program = parse_suite(contents)?; + let stmt = program.first().unwrap(); + let locator = Locator::new(contents); + assert!(!has_trailing_content(stmt.end(), &locator)); + + let contents = "x = 1; y = 2"; + let program = parse_suite(contents)?; + let stmt = program.first().unwrap(); + let locator = Locator::new(contents); + assert!(has_trailing_content(stmt.end(), &locator)); + + let contents = "x = 1 "; + let program = parse_suite(contents)?; + let stmt = program.first().unwrap(); + let locator = Locator::new(contents); + assert!(!has_trailing_content(stmt.end(), &locator)); + + let contents = "x = 1 # Comment"; + let program = parse_suite(contents)?; + let stmt = program.first().unwrap(); + let locator = Locator::new(contents); + assert!(!has_trailing_content(stmt.end(), &locator)); + + let contents = r" +x = 1 +y = 2 +" + .trim(); + let program = parse_suite(contents)?; + let stmt = program.first().unwrap(); + let locator = Locator::new(contents); + assert!(!has_trailing_content(stmt.end(), &locator)); + + Ok(()) +} diff --git a/crates/ruff_source_file/Cargo.toml b/crates/ruff_source_file/Cargo.toml index c6910e4fa4..f3cc40807a 100644 --- a/crates/ruff_source_file/Cargo.toml +++ b/crates/ruff_source_file/Cargo.toml @@ -13,9 +13,10 @@ license = { workspace = true } [lib] [dependencies] +ruff_text_size = { path = "../ruff_text_size" } + memchr = { workspace = true } once_cell = { workspace = true } -ruff_text_size = { path = "../ruff_text_size" } serde = { workspace = true, optional = true } [dev-dependencies]