diff --git a/crates/red_knot_python_semantic/resources/mdtest/comprehensions/basic.md b/crates/red_knot_python_semantic/resources/mdtest/comprehensions/basic.md new file mode 100644 index 0000000000..5236b27cce --- /dev/null +++ b/crates/red_knot_python_semantic/resources/mdtest/comprehensions/basic.md @@ -0,0 +1,149 @@ +# Comprehensions + +## Basic comprehensions + +```py +class IntIterator: + def __next__(self) -> int: + return 42 + +class IntIterable: + def __iter__(self) -> IntIterator: + return IntIterator() + +# revealed: int +[reveal_type(x) for x in IntIterable()] + +class IteratorOfIterables: + def __next__(self) -> IntIterable: + return IntIterable() + +class IterableOfIterables: + def __iter__(self) -> IteratorOfIterables: + return IteratorOfIterables() + +# revealed: tuple[int, IntIterable] +[reveal_type((x, y)) for y in IterableOfIterables() for x in y] + +# revealed: int +{reveal_type(x): 0 for x in IntIterable()} + +# revealed: int +{0: reveal_type(x) for x in IntIterable()} +``` + +## Nested comprehension + +```py +class IntIterator: + def __next__(self) -> int: + return 42 + +class IntIterable: + def __iter__(self) -> IntIterator: + return IntIterator() + +# revealed: tuple[int, int] +[[reveal_type((x, y)) for x in IntIterable()] for y in IntIterable()] +``` + +## Comprehension referencing outer comprehension + +```py +class IntIterator: + def __next__(self) -> int: + return 42 + +class IntIterable: + def __iter__(self) -> IntIterator: + return IntIterator() + +class IteratorOfIterables: + def __next__(self) -> IntIterable: + return IntIterable() + +class IterableOfIterables: + def __iter__(self) -> IteratorOfIterables: + return IteratorOfIterables() + +# revealed: tuple[int, IntIterable] +[[reveal_type((x, y)) for x in y] for y in IterableOfIterables()] +``` + +## Comprehension with unbound iterable + +Iterating over an unbound iterable yields `Unknown`: + +```py +# error: [unresolved-reference] "Name `x` used when not defined" +# revealed: Unknown +[reveal_type(z) for z in x] + +class IntIterator: + def __next__(self) -> int: + return 42 + +class IntIterable: + def __iter__(self) -> IntIterator: + return IntIterator() + +# error: [not-iterable] "Object of type `int` is not iterable" +# revealed: tuple[int, Unknown] +[reveal_type((x, z)) for x in IntIterable() for z in x] +``` + +## Starred expressions + +Starred expressions must be iterable + +```py +class NotIterable: ... + +class Iterator: + def __next__(self) -> int: + return 42 + +class Iterable: + def __iter__(self) -> Iterator: ... + +# This is fine: +x = [*Iterable()] + +# error: [not-iterable] "Object of type `NotIterable` is not iterable" +y = [*NotIterable()] +``` + +## Async comprehensions + +### Basic + +```py +class AsyncIterator: + async def __anext__(self) -> int: + return 42 + +class AsyncIterable: + def __aiter__(self) -> AsyncIterator: + return AsyncIterator() + +# revealed: @Todo(async iterables/iterators) +[reveal_type(x) async for x in AsyncIterable()] +``` + +### Invalid async comprehension + +This tests that we understand that `async` comprehensions do *not* work according to the synchronous +iteration protocol + +```py +class Iterator: + def __next__(self) -> int: + return 42 + +class Iterable: + def __iter__(self) -> Iterator: + return Iterator() + +# revealed: @Todo(async iterables/iterators) +[reveal_type(x) async for x in Iterable()] +``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/comprehensions/invalid_syntax.md b/crates/red_knot_python_semantic/resources/mdtest/comprehensions/invalid_syntax.md new file mode 100644 index 0000000000..20c71be171 --- /dev/null +++ b/crates/red_knot_python_semantic/resources/mdtest/comprehensions/invalid_syntax.md @@ -0,0 +1,43 @@ +# Comprehensions with invalid syntax + +```py +class IntIterator: + def __next__(self) -> int: + return 42 + +class IntIterable: + def __iter__(self) -> IntIterator: + return IntIterator() + +# Missing 'in' keyword. + +# It's reasonably clear here what they *meant* to write, +# so we'll still infer the correct type: + +# error: [invalid-syntax] "Expected 'in', found name" +# revealed: int +[reveal_type(a) for a IntIterable()] + + +# Missing iteration variable + +# error: [invalid-syntax] "Expected an identifier, but found a keyword 'in' that cannot be used here" +# error: [invalid-syntax] "Expected 'in', found name" +# error: [unresolved-reference] +# revealed: Unknown +[reveal_type(b) for in IntIterable()] + + +# Missing iterable + +# error: [invalid-syntax] "Expected an expression" +# revealed: Unknown +[reveal_type(c) for c in] + + +# Missing 'in' keyword and missing iterable + +# error: [invalid-syntax] "Expected 'in', found ']'" +# revealed: Unknown +[reveal_type(d) for d] +``` diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index e5e1906763..b4a2ec68af 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -6007,7 +6007,6 @@ mod tests { use crate::semantic_index::definition::Definition; use crate::semantic_index::symbol::FileScopeId; use crate::semantic_index::{global_scope, semantic_index, symbol_table, use_def_map}; - use crate::types::check_types; use ruff_db::files::{system_path_to_file, File}; use ruff_db::system::DbWithTestSystem; use ruff_db::testing::assert_function_query_was_not_run; @@ -6050,35 +6049,6 @@ mod tests { symbol(db, scope, symbol_name) } - #[track_caller] - fn assert_scope_type( - db: &TestDb, - file_name: &str, - scopes: &[&str], - symbol_name: &str, - expected: &str, - ) { - let ty = get_symbol(db, file_name, scopes, symbol_name).expect_type(); - assert_eq!(ty.display(db).to_string(), expected); - } - - #[track_caller] - fn assert_diagnostic_messages(diagnostics: &TypeCheckDiagnostics, expected: &[&str]) { - let messages: Vec<&str> = diagnostics - .iter() - .map(|diagnostic| diagnostic.message()) - .collect(); - assert_eq!(&messages, expected); - } - - #[track_caller] - fn assert_file_diagnostics(db: &TestDb, filename: &str, expected: &[&str]) { - let file = system_path_to_file(db, filename).unwrap(); - let diagnostics = check_types(db, file); - - assert_diagnostic_messages(diagnostics, expected); - } - #[test] fn not_literal_string() -> anyhow::Result<()> { let mut db = setup_db(); @@ -6205,385 +6175,6 @@ mod tests { Ok(()) } - #[test] - fn basic_comprehension() -> anyhow::Result<()> { - let mut db = setup_db(); - - db.write_dedented( - "src/a.py", - " - def foo(): - [x for y in IterableOfIterables() for x in y] - - class IntIterator: - def __next__(self) -> int: - return 42 - - class IntIterable: - def __iter__(self) -> IntIterator: - return IntIterator() - - class IteratorOfIterables: - def __next__(self) -> IntIterable: - return IntIterable() - - class IterableOfIterables: - def __iter__(self) -> IteratorOfIterables: - return IteratorOfIterables() - ", - )?; - - assert_scope_type(&db, "src/a.py", &["foo", ""], "x", "int"); - assert_scope_type(&db, "src/a.py", &["foo", ""], "y", "IntIterable"); - - Ok(()) - } - - #[test] - fn comprehension_inside_comprehension() -> anyhow::Result<()> { - let mut db = setup_db(); - - db.write_dedented( - "src/a.py", - " - def foo(): - [[x for x in iter1] for y in iter2] - - class IntIterator: - def __next__(self) -> int: - return 42 - - class IntIterable: - def __iter__(self) -> IntIterator: - return IntIterator() - - iter1 = IntIterable() - iter2 = IntIterable() - ", - )?; - - assert_scope_type( - &db, - "src/a.py", - &["foo", "", ""], - "x", - "int", - ); - assert_scope_type(&db, "src/a.py", &["foo", ""], "y", "int"); - - Ok(()) - } - - #[test] - fn inner_comprehension_referencing_outer_comprehension() -> anyhow::Result<()> { - let mut db = setup_db(); - - db.write_dedented( - "src/a.py", - " - def foo(): - [[x for x in y] for y in z] - - class IntIterator: - def __next__(self) -> int: - return 42 - - class IntIterable: - def __iter__(self) -> IntIterator: - return IntIterator() - - class IteratorOfIterables: - def __next__(self) -> IntIterable: - return IntIterable() - - class IterableOfIterables: - def __iter__(self) -> IteratorOfIterables: - return IteratorOfIterables() - - z = IterableOfIterables() - ", - )?; - - assert_scope_type( - &db, - "src/a.py", - &["foo", "", ""], - "x", - "int", - ); - assert_scope_type(&db, "src/a.py", &["foo", ""], "y", "IntIterable"); - - Ok(()) - } - - #[test] - fn comprehension_with_unbound_iter() -> anyhow::Result<()> { - let mut db = setup_db(); - - db.write_dedented("src/a.py", "[z for z in x]")?; - - let x = get_symbol(&db, "src/a.py", &[""], "x"); - assert!(x.is_unbound()); - - // Iterating over an unbound iterable yields `Unknown`: - assert_scope_type(&db, "src/a.py", &[""], "z", "Unknown"); - - assert_file_diagnostics(&db, "src/a.py", &["Name `x` used when not defined"]); - - Ok(()) - } - - #[test] - fn comprehension_with_not_iterable_iter_in_second_comprehension() -> anyhow::Result<()> { - let mut db = setup_db(); - - db.write_dedented( - "src/a.py", - " - def foo(): - [z for x in IntIterable() for z in x] - - class IntIterator: - def __next__(self) -> int: - return 42 - - class IntIterable: - def __iter__(self) -> IntIterator: - return IntIterator() - ", - )?; - - assert_scope_type(&db, "src/a.py", &["foo", ""], "x", "int"); - assert_scope_type(&db, "src/a.py", &["foo", ""], "z", "Unknown"); - assert_file_diagnostics(&db, "src/a.py", &["Object of type `int` is not iterable"]); - - Ok(()) - } - - #[test] - fn dict_comprehension_variable_key() -> anyhow::Result<()> { - let mut db = setup_db(); - - db.write_dedented( - "src/a.py", - " - def foo(): - {x: 0 for x in IntIterable()} - - class IntIterator: - def __next__(self) -> int: - return 42 - - class IntIterable: - def __iter__(self) -> IntIterator: - return IntIterator() - ", - )?; - - assert_scope_type(&db, "src/a.py", &["foo", ""], "x", "int"); - assert_file_diagnostics(&db, "src/a.py", &[]); - - Ok(()) - } - - #[test] - fn dict_comprehension_variable_value() -> anyhow::Result<()> { - let mut db = setup_db(); - - db.write_dedented( - "src/a.py", - " - def foo(): - {0: x for x in IntIterable()} - - class IntIterator: - def __next__(self) -> int: - return 42 - - class IntIterable: - def __iter__(self) -> IntIterator: - return IntIterator() - ", - )?; - - assert_scope_type(&db, "src/a.py", &["foo", ""], "x", "int"); - assert_file_diagnostics(&db, "src/a.py", &[]); - - Ok(()) - } - - #[test] - fn comprehension_with_missing_in_keyword() -> anyhow::Result<()> { - let mut db = setup_db(); - - db.write_dedented( - "src/a.py", - " - def foo(): - [z for z IntIterable()] - - class IntIterator: - def __next__(self) -> int: - return 42 - - class IntIterable: - def __iter__(self) -> IntIterator: - return IntIterator() - ", - )?; - - // We'll emit a diagnostic separately for invalid syntax, - // but it's reasonably clear here what they *meant* to write, - // so we'll still infer the correct type: - assert_scope_type(&db, "src/a.py", &["foo", ""], "z", "int"); - Ok(()) - } - - #[test] - fn comprehension_with_missing_iter() -> anyhow::Result<()> { - let mut db = setup_db(); - - db.write_dedented( - "src/a.py", - " - def foo(): - [z for in IntIterable()] - - class IntIterator: - def __next__(self) -> int: - return 42 - - class IntIterable: - def __iter__(self) -> IntIterator: - return IntIterator() - ", - )?; - - let z = get_symbol(&db, "src/a.py", &["foo", ""], "z"); - assert!(z.is_unbound()); - - // (There is a diagnostic for invalid syntax that's emitted, but it's not listed by `assert_file_diagnostics`) - assert_file_diagnostics(&db, "src/a.py", &["Name `z` used when not defined"]); - - Ok(()) - } - - #[test] - fn comprehension_with_missing_for() -> anyhow::Result<()> { - let mut db = setup_db(); - db.write_dedented("src/a.py", "[z for z in]")?; - assert_scope_type(&db, "src/a.py", &[""], "z", "Unknown"); - Ok(()) - } - - #[test] - fn comprehension_with_missing_in_keyword_and_missing_iter() -> anyhow::Result<()> { - let mut db = setup_db(); - db.write_dedented("src/a.py", "[z for z]")?; - assert_scope_type(&db, "src/a.py", &[""], "z", "Unknown"); - Ok(()) - } - - /// This tests that we understand that `async` comprehensions - /// do not work according to the synchronous iteration protocol - #[test] - fn invalid_async_comprehension() -> anyhow::Result<()> { - let mut db = setup_db(); - - db.write_dedented( - "src/a.py", - " - async def foo(): - [x async for x in Iterable()] - class Iterator: - def __next__(self) -> int: - return 42 - class Iterable: - def __iter__(self) -> Iterator: - return Iterator() - ", - )?; - - // We currently return `Todo` for all async comprehensions, - // including comprehensions that have invalid syntax - assert_scope_type( - &db, - "src/a.py", - &["foo", ""], - "x", - if cfg!(debug_assertions) { - "@Todo(async iterables/iterators)" - } else { - "@Todo" - }, - ); - - Ok(()) - } - - #[test] - fn basic_async_comprehension() -> anyhow::Result<()> { - let mut db = setup_db(); - - db.write_dedented( - "src/a.py", - " - async def foo(): - [x async for x in AsyncIterable()] - class AsyncIterator: - async def __anext__(self) -> int: - return 42 - class AsyncIterable: - def __aiter__(self) -> AsyncIterator: - return AsyncIterator() - ", - )?; - - // TODO async iterables/iterators! --Alex - assert_scope_type( - &db, - "src/a.py", - &["foo", ""], - "x", - if cfg!(debug_assertions) { - "@Todo(async iterables/iterators)" - } else { - "@Todo" - }, - ); - - Ok(()) - } - - #[test] - fn starred_expressions_must_be_iterable() { - let mut db = setup_db(); - - db.write_dedented( - "src/a.py", - " - class NotIterable: pass - - class Iterator: - def __next__(self) -> int: - return 42 - - class Iterable: - def __iter__(self) -> Iterator: ... - - x = [*NotIterable()] - y = [*Iterable()] - ", - ) - .unwrap(); - - assert_file_diagnostics( - &db, - "/src/a.py", - &["Object of type `NotIterable` is not iterable"], - ); - } - #[test] fn pep695_type_params() { let mut db = setup_db();