From ca4fdf452d61387a6b39ec44ea6bc3607ad3fda5 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Tue, 29 Apr 2025 09:03:06 -0400 Subject: [PATCH] Create `TypeVarInstance` type for legacy typevars (#16538) We are currently representing type variables using a `KnownInstance` variant, which wraps a `TypeVarInstance` that contains the information about the typevar (name, bounds, constraints, default type). We were previously only constructing that type for PEP 695 typevars. This PR constructs that type for legacy typevars as well. It also detects functions that are generic because they use legacy typevars in their parameter list. With the existing logic for inferring specializations of function calls (#17301), that means that we are correctly detecting that the definition of `reveal_type` in the typeshed is generic, and inferring the correct specialization of `_T` for each call site. This does not yet handle legacy generic classes; that will come in a follow-on PR. --- .../resources/mdtest/function/return_type.md | 2 +- .../resources/mdtest/generics/functions.md | 33 +++ .../resources/mdtest/generics/legacy.md | 58 +++- .../resources/mdtest/generics/pep695.md | 43 ++- .../resources/mdtest/generics/scoping.md | 3 +- ...functions_-_Inferring_a_bound_typevar.snap | 86 ++++++ ...ons_-_Inferring_a_constrained_typevar.snap | 101 +++++++ ...ion_return_type_-_Invalid_return_type.snap | 13 +- .../resources/mdtest/subscript/lists.md | 2 +- .../type_properties/is_assignable_to.md | 2 + .../src/semantic_index/builder.rs | 25 +- .../src/semantic_index/expression.rs | 11 + crates/red_knot_python_semantic/src/types.rs | 198 ++++++++++++- .../src/types/call/bind.rs | 208 +++++++++----- .../src/types/diagnostic.rs | 33 ++- .../src/types/display.rs | 2 +- .../src/types/generics.rs | 126 +++++++-- .../src/types/infer.rs | 263 +++++++++++++----- .../src/types/known_instance.rs | 11 +- .../src/types/signatures.rs | 38 ++- crates/ruff_benchmark/benches/red_knot.rs | 23 +- knot.schema.json | 10 + 22 files changed, 1100 insertions(+), 191 deletions(-) create mode 100644 crates/red_knot_python_semantic/resources/mdtest/snapshots/functions.md_-_Generic_functions_-_Inferring_a_bound_typevar.snap create mode 100644 crates/red_knot_python_semantic/resources/mdtest/snapshots/functions.md_-_Generic_functions_-_Inferring_a_constrained_typevar.snap diff --git a/crates/red_knot_python_semantic/resources/mdtest/function/return_type.md b/crates/red_knot_python_semantic/resources/mdtest/function/return_type.md index 1365807e95..8e83d03654 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/function/return_type.md +++ b/crates/red_knot_python_semantic/resources/mdtest/function/return_type.md @@ -203,7 +203,7 @@ from typing import TypeVar T = TypeVar("T") -# TODO: `invalid-return-type` error should be emitted +# error: [invalid-return-type] def m(x: T) -> T: ... ``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/generics/functions.md b/crates/red_knot_python_semantic/resources/mdtest/generics/functions.md index 10732bf3a2..22c62b1d35 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/generics/functions.md +++ b/crates/red_knot_python_semantic/resources/mdtest/generics/functions.md @@ -71,6 +71,39 @@ def f[T](x: list[T]) -> T: reveal_type(f([1.0, 2.0])) # revealed: Unknown ``` +## Inferring a bound typevar + + + +```py +from typing_extensions import reveal_type + +def f[T: int](x: T) -> T: + return x + +reveal_type(f(1)) # revealed: Literal[1] +reveal_type(f(True)) # revealed: Literal[True] +# error: [invalid-argument-type] "Argument to this function is incorrect: Argument type `Literal["string"]` does not satisfy upper bound of type variable `T`" +reveal_type(f("string")) # revealed: Unknown +``` + +## Inferring a constrained typevar + + + +```py +from typing_extensions import reveal_type + +def f[T: (int, None)](x: T) -> T: + return x + +reveal_type(f(1)) # revealed: int +reveal_type(f(True)) # revealed: int +reveal_type(f(None)) # revealed: None +# error: [invalid-argument-type] "Argument to this function is incorrect: Argument type `Literal["string"]` does not satisfy constraints of type variable `T`" +reveal_type(f("string")) # revealed: Unknown +``` + ## Typevar constraints If a type parameter has an upper bound, that upper bound constrains which types can be used for that diff --git a/crates/red_knot_python_semantic/resources/mdtest/generics/legacy.md b/crates/red_knot_python_semantic/resources/mdtest/generics/legacy.md index 6aea25ed82..17021dd398 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/generics/legacy.md +++ b/crates/red_knot_python_semantic/resources/mdtest/generics/legacy.md @@ -19,6 +19,9 @@ in newer Python releases. from typing import TypeVar T = TypeVar("T") +reveal_type(type(T)) # revealed: Literal[TypeVar] +reveal_type(T) # revealed: typing.TypeVar +reveal_type(T.__name__) # revealed: Literal["T"] ``` ### Directly assigned to a variable @@ -29,7 +32,12 @@ T = TypeVar("T") ```py from typing import TypeVar -# TODO: error +T = TypeVar("T") +# TODO: no error +# error: [invalid-legacy-type-variable] +U: TypeVar = TypeVar("U") + +# error: [invalid-legacy-type-variable] "A legacy `typing.TypeVar` must be immediately assigned to a variable" TestList = list[TypeVar("W")] ``` @@ -40,7 +48,7 @@ TestList = list[TypeVar("W")] ```py from typing import TypeVar -# TODO: error +# error: [invalid-legacy-type-variable] "The name of a legacy `typing.TypeVar` (`Q`) must match the name of the variable it is assigned to (`T`)" T = TypeVar("Q") ``` @@ -57,6 +65,52 @@ T = TypeVar("T") T = TypeVar("T") ``` +### Type variables with a default + +Note that the `__default__` property is only available in Python ≥3.13. + +```toml +[environment] +python-version = "3.13" +``` + +```py +from typing import TypeVar + +T = TypeVar("T", default=int) +reveal_type(T.__default__) # revealed: int +reveal_type(T.__bound__) # revealed: None +reveal_type(T.__constraints__) # revealed: tuple[()] + +S = TypeVar("S") +reveal_type(S.__default__) # revealed: NoDefault +``` + +### Type variables with an upper bound + +```py +from typing import TypeVar + +T = TypeVar("T", bound=int) +reveal_type(T.__bound__) # revealed: int +reveal_type(T.__constraints__) # revealed: tuple[()] + +S = TypeVar("S") +reveal_type(S.__bound__) # revealed: None +``` + +### Type variables with constraints + +```py +from typing import TypeVar + +T = TypeVar("T", int, str) +reveal_type(T.__constraints__) # revealed: tuple[int, str] + +S = TypeVar("S") +reveal_type(S.__constraints__) # revealed: tuple[()] +``` + ### Cannot have only one constraint > `TypeVar` supports constraining parametric types to a fixed set of possible types...There should diff --git a/crates/red_knot_python_semantic/resources/mdtest/generics/pep695.md b/crates/red_knot_python_semantic/resources/mdtest/generics/pep695.md index 67e69940f3..67a4fdc4f0 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/generics/pep695.md +++ b/crates/red_knot_python_semantic/resources/mdtest/generics/pep695.md @@ -17,10 +17,51 @@ instances of `typing.TypeVar`, just like legacy type variables. ```py def f[T](): reveal_type(type(T)) # revealed: Literal[TypeVar] - reveal_type(T) # revealed: T + reveal_type(T) # revealed: typing.TypeVar reveal_type(T.__name__) # revealed: Literal["T"] ``` +### Type variables with a default + +Note that the `__default__` property is only available in Python ≥3.13. + +```toml +[environment] +python-version = "3.13" +``` + +```py +def f[T = int](): + reveal_type(T.__default__) # revealed: int + reveal_type(T.__bound__) # revealed: None + reveal_type(T.__constraints__) # revealed: tuple[()] + +def g[S](): + reveal_type(S.__default__) # revealed: NoDefault +``` + +### Type variables with an upper bound + +```py +def f[T: int](): + reveal_type(T.__bound__) # revealed: int + reveal_type(T.__constraints__) # revealed: tuple[()] + +def g[S](): + reveal_type(S.__bound__) # revealed: None +``` + +### Type variables with constraints + +```py +def f[T: (int, str)](): + reveal_type(T.__constraints__) # revealed: tuple[int, str] + reveal_type(T.__bound__) # revealed: None + +def g[S](): + reveal_type(S.__constraints__) # revealed: tuple[()] +``` + ### Cannot have only one constraint > `TypeVar` supports constraining parametric types to a fixed set of possible types...There should diff --git a/crates/red_knot_python_semantic/resources/mdtest/generics/scoping.md b/crates/red_knot_python_semantic/resources/mdtest/generics/scoping.md index 9712be7213..0300cadc87 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/generics/scoping.md +++ b/crates/red_knot_python_semantic/resources/mdtest/generics/scoping.md @@ -142,8 +142,7 @@ class Legacy(Generic[T]): return y legacy: Legacy[int] = Legacy() -# TODO: revealed: str -reveal_type(legacy.m(1, "string")) # revealed: @Todo(Support for `typing.TypeVar` instances in type expressions) +reveal_type(legacy.m(1, "string")) # revealed: Literal["string"] ``` With PEP 695 syntax, it is clearer that the method uses a separate typevar: diff --git a/crates/red_knot_python_semantic/resources/mdtest/snapshots/functions.md_-_Generic_functions_-_Inferring_a_bound_typevar.snap b/crates/red_knot_python_semantic/resources/mdtest/snapshots/functions.md_-_Generic_functions_-_Inferring_a_bound_typevar.snap new file mode 100644 index 0000000000..1fcfed3058 --- /dev/null +++ b/crates/red_knot_python_semantic/resources/mdtest/snapshots/functions.md_-_Generic_functions_-_Inferring_a_bound_typevar.snap @@ -0,0 +1,86 @@ +--- +source: crates/red_knot_test/src/lib.rs +expression: snapshot +--- +--- +mdtest name: functions.md - Generic functions - Inferring a bound typevar +mdtest path: crates/red_knot_python_semantic/resources/mdtest/generics/functions.md +--- + +# Python source files + +## mdtest_snippet.py + +``` +1 | from typing_extensions import reveal_type +2 | +3 | def f[T: int](x: T) -> T: +4 | return x +5 | +6 | reveal_type(f(1)) # revealed: Literal[1] +7 | reveal_type(f(True)) # revealed: Literal[True] +8 | # error: [invalid-argument-type] "Argument to this function is incorrect: Argument type `Literal["string"]` does not satisfy upper bound of type variable `T`" +9 | reveal_type(f("string")) # revealed: Unknown +``` + +# Diagnostics + +``` +info: revealed-type: Revealed type + --> src/mdtest_snippet.py:6:1 + | +4 | return x +5 | +6 | reveal_type(f(1)) # revealed: Literal[1] + | ^^^^^^^^^^^^^^^^^ `Literal[1]` +7 | reveal_type(f(True)) # revealed: Literal[True] +8 | # error: [invalid-argument-type] "Argument to this function is incorrect: Argument type `Literal["string"]` does not satisfy upper bo... + | + +``` + +``` +info: revealed-type: Revealed type + --> src/mdtest_snippet.py:7:1 + | +6 | reveal_type(f(1)) # revealed: Literal[1] +7 | reveal_type(f(True)) # revealed: Literal[True] + | ^^^^^^^^^^^^^^^^^^^^ `Literal[True]` +8 | # error: [invalid-argument-type] "Argument to this function is incorrect: Argument type `Literal["string"]` does not satisfy upper b... +9 | reveal_type(f("string")) # revealed: Unknown + | + +``` + +``` +error: lint:invalid-argument-type: Argument to this function is incorrect + --> src/mdtest_snippet.py:9:15 + | +7 | reveal_type(f(True)) # revealed: Literal[True] +8 | # error: [invalid-argument-type] "Argument to this function is incorrect: Argument type `Literal["string"]` does not satisfy upper b... +9 | reveal_type(f("string")) # revealed: Unknown + | ^^^^^^^^ Argument type `Literal["string"]` does not satisfy upper bound of type variable `T` + | +info: Type variable defined here + --> src/mdtest_snippet.py:3:7 + | +1 | from typing_extensions import reveal_type +2 | +3 | def f[T: int](x: T) -> T: + | ^^^^^^ +4 | return x + | + +``` + +``` +info: revealed-type: Revealed type + --> src/mdtest_snippet.py:9:1 + | +7 | reveal_type(f(True)) # revealed: Literal[True] +8 | # error: [invalid-argument-type] "Argument to this function is incorrect: Argument type `Literal["string"]` does not satisfy upper b... +9 | reveal_type(f("string")) # revealed: Unknown + | ^^^^^^^^^^^^^^^^^^^^^^^^ `Unknown` + | + +``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/snapshots/functions.md_-_Generic_functions_-_Inferring_a_constrained_typevar.snap b/crates/red_knot_python_semantic/resources/mdtest/snapshots/functions.md_-_Generic_functions_-_Inferring_a_constrained_typevar.snap new file mode 100644 index 0000000000..2e2802007b --- /dev/null +++ b/crates/red_knot_python_semantic/resources/mdtest/snapshots/functions.md_-_Generic_functions_-_Inferring_a_constrained_typevar.snap @@ -0,0 +1,101 @@ +--- +source: crates/red_knot_test/src/lib.rs +expression: snapshot +--- +--- +mdtest name: functions.md - Generic functions - Inferring a constrained typevar +mdtest path: crates/red_knot_python_semantic/resources/mdtest/generics/functions.md +--- + +# Python source files + +## mdtest_snippet.py + +``` + 1 | from typing_extensions import reveal_type + 2 | + 3 | def f[T: (int, None)](x: T) -> T: + 4 | return x + 5 | + 6 | reveal_type(f(1)) # revealed: int + 7 | reveal_type(f(True)) # revealed: int + 8 | reveal_type(f(None)) # revealed: None + 9 | # error: [invalid-argument-type] "Argument to this function is incorrect: Argument type `Literal["string"]` does not satisfy constraints of type variable `T`" +10 | reveal_type(f("string")) # revealed: Unknown +``` + +# Diagnostics + +``` +info: revealed-type: Revealed type + --> src/mdtest_snippet.py:6:1 + | +4 | return x +5 | +6 | reveal_type(f(1)) # revealed: int + | ^^^^^^^^^^^^^^^^^ `int` +7 | reveal_type(f(True)) # revealed: int +8 | reveal_type(f(None)) # revealed: None + | + +``` + +``` +info: revealed-type: Revealed type + --> src/mdtest_snippet.py:7:1 + | +6 | reveal_type(f(1)) # revealed: int +7 | reveal_type(f(True)) # revealed: int + | ^^^^^^^^^^^^^^^^^^^^ `int` +8 | reveal_type(f(None)) # revealed: None +9 | # error: [invalid-argument-type] "Argument to this function is incorrect: Argument type `Literal["string"]` does not satisfy constra... + | + +``` + +``` +info: revealed-type: Revealed type + --> src/mdtest_snippet.py:8:1 + | + 6 | reveal_type(f(1)) # revealed: int + 7 | reveal_type(f(True)) # revealed: int + 8 | reveal_type(f(None)) # revealed: None + | ^^^^^^^^^^^^^^^^^^^^ `None` + 9 | # error: [invalid-argument-type] "Argument to this function is incorrect: Argument type `Literal["string"]` does not satisfy constra... +10 | reveal_type(f("string")) # revealed: Unknown + | + +``` + +``` +error: lint:invalid-argument-type: Argument to this function is incorrect + --> src/mdtest_snippet.py:10:15 + | + 8 | reveal_type(f(None)) # revealed: None + 9 | # error: [invalid-argument-type] "Argument to this function is incorrect: Argument type `Literal["string"]` does not satisfy constra... +10 | reveal_type(f("string")) # revealed: Unknown + | ^^^^^^^^ Argument type `Literal["string"]` does not satisfy constraints of type variable `T` + | +info: Type variable defined here + --> src/mdtest_snippet.py:3:7 + | +1 | from typing_extensions import reveal_type +2 | +3 | def f[T: (int, None)](x: T) -> T: + | ^^^^^^^^^^^^^^ +4 | return x + | + +``` + +``` +info: revealed-type: Revealed type + --> src/mdtest_snippet.py:10:1 + | + 8 | reveal_type(f(None)) # revealed: None + 9 | # error: [invalid-argument-type] "Argument to this function is incorrect: Argument type `Literal["string"]` does not satisfy constra... +10 | reveal_type(f("string")) # revealed: Unknown + | ^^^^^^^^^^^^^^^^^^^^^^^^ `Unknown` + | + +``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/snapshots/return_type.md_-_Function_return_type_-_Invalid_return_type.snap b/crates/red_knot_python_semantic/resources/mdtest/snapshots/return_type.md_-_Function_return_type_-_Invalid_return_type.snap index 67a88e4dc8..7e439527f1 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/snapshots/return_type.md_-_Function_return_type_-_Invalid_return_type.snap +++ b/crates/red_knot_python_semantic/resources/mdtest/snapshots/return_type.md_-_Function_return_type_-_Invalid_return_type.snap @@ -28,7 +28,7 @@ mdtest path: crates/red_knot_python_semantic/resources/mdtest/function/return_ty 14 | 15 | T = TypeVar("T") 16 | -17 | # TODO: `invalid-return-type` error should be emitted +17 | # error: [invalid-return-type] 18 | def m(x: T) -> T: ... ``` @@ -79,3 +79,14 @@ error: lint:invalid-return-type: Return type does not match returned value | ``` + +``` +error: lint:invalid-return-type: Function can implicitly return `None`, which is not assignable to return type `T` + --> src/mdtest_snippet.py:18:16 + | +17 | # error: [invalid-return-type] +18 | def m(x: T) -> T: ... + | ^ + | + +``` diff --git a/crates/red_knot_python_semantic/resources/mdtest/subscript/lists.md b/crates/red_knot_python_semantic/resources/mdtest/subscript/lists.md index 192dc4a88e..d074d1b826 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/subscript/lists.md +++ b/crates/red_knot_python_semantic/resources/mdtest/subscript/lists.md @@ -12,7 +12,7 @@ x = [1, 2, 3] reveal_type(x) # revealed: list # TODO reveal int -reveal_type(x[0]) # revealed: @Todo(Support for `typing.TypeVar` instances in type expressions) +reveal_type(x[0]) # revealed: Unknown # TODO reveal list reveal_type(x[0:1]) # revealed: @Todo(specialized non-generic class) diff --git a/crates/red_knot_python_semantic/resources/mdtest/type_properties/is_assignable_to.md b/crates/red_knot_python_semantic/resources/mdtest/type_properties/is_assignable_to.md index ce7198ba69..f8b8adf5f8 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/type_properties/is_assignable_to.md +++ b/crates/red_knot_python_semantic/resources/mdtest/type_properties/is_assignable_to.md @@ -583,6 +583,8 @@ from functools import partial def f(x: int, y: str) -> None: ... +# TODO: no error +# error: [invalid-assignment] "Object of type `partial` is not assignable to `(int, /) -> None`" c1: Callable[[int], None] = partial(f, y="a") ``` diff --git a/crates/red_knot_python_semantic/src/semantic_index/builder.rs b/crates/red_knot_python_semantic/src/semantic_index/builder.rs index a17e4523f3..22af0fda06 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/builder.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/builder.rs @@ -754,19 +754,35 @@ impl<'db> SemanticIndexBuilder<'db> { /// Record an expression that needs to be a Salsa ingredient, because we need to infer its type /// standalone (type narrowing tests, RHS of an assignment.) fn add_standalone_expression(&mut self, expression_node: &ast::Expr) -> Expression<'db> { - self.add_standalone_expression_impl(expression_node, ExpressionKind::Normal) + self.add_standalone_expression_impl(expression_node, ExpressionKind::Normal, None) + } + + /// Record an expression that is immediately assigned to a target, and that needs to be a Salsa + /// ingredient, because we need to infer its type standalone (type narrowing tests, RHS of an + /// assignment.) + fn add_standalone_assigned_expression( + &mut self, + expression_node: &ast::Expr, + assigned_to: &ast::StmtAssign, + ) -> Expression<'db> { + self.add_standalone_expression_impl( + expression_node, + ExpressionKind::Normal, + Some(assigned_to), + ) } /// Same as [`SemanticIndexBuilder::add_standalone_expression`], but marks the expression as a /// *type* expression, which makes sure that it will later be inferred as such. fn add_standalone_type_expression(&mut self, expression_node: &ast::Expr) -> Expression<'db> { - self.add_standalone_expression_impl(expression_node, ExpressionKind::TypeExpression) + self.add_standalone_expression_impl(expression_node, ExpressionKind::TypeExpression, None) } fn add_standalone_expression_impl( &mut self, expression_node: &ast::Expr, expression_kind: ExpressionKind, + assigned_to: Option<&ast::StmtAssign>, ) -> Expression<'db> { let expression = Expression::new( self.db, @@ -776,6 +792,9 @@ impl<'db> SemanticIndexBuilder<'db> { unsafe { AstNodeRef::new(self.module.clone(), expression_node) }, + #[allow(unsafe_code)] + assigned_to + .map(|assigned_to| unsafe { AstNodeRef::new(self.module.clone(), assigned_to) }), expression_kind, countme::Count::default(), ); @@ -1377,7 +1396,7 @@ where debug_assert_eq!(&self.current_assignments, &[]); self.visit_expr(&node.value); - let value = self.add_standalone_expression(&node.value); + let value = self.add_standalone_assigned_expression(&node.value, node); for target in &node.targets { self.add_unpackable_assignment(&Unpackable::Assign(node), target, value); diff --git a/crates/red_knot_python_semantic/src/semantic_index/expression.rs b/crates/red_knot_python_semantic/src/semantic_index/expression.rs index 9ac1fd30b8..8c50ea5bb9 100644 --- a/crates/red_knot_python_semantic/src/semantic_index/expression.rs +++ b/crates/red_knot_python_semantic/src/semantic_index/expression.rs @@ -44,6 +44,17 @@ pub(crate) struct Expression<'db> { #[return_ref] pub(crate) node_ref: AstNodeRef, + /// An assignment statement, if this expression is immediately used as the rhs of that + /// assignment. + /// + /// (Note that this is the _immediately_ containing assignment — if a complex expression is + /// assigned to some target, only the outermost expression node has this set. The inner + /// expressions are used to build up the assignment result, and are not "immediately assigned" + /// to the target, and so have `None` for this field.) + #[no_eq] + #[tracked] + pub(crate) assigned_to: Option>, + /// Should this expression be inferred as a normal expression or a type expression? pub(crate) kind: ExpressionKind, diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 1c2e15a3cd..adc6ac478e 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -348,6 +348,19 @@ impl<'db> PropertyInstanceType<'db> { .map(|ty| ty.apply_specialization(db, specialization)); Self::new(db, getter, setter) } + + fn find_legacy_typevars( + self, + db: &'db dyn Db, + typevars: &mut FxOrderSet>, + ) { + if let Some(ty) = self.getter(db) { + ty.find_legacy_typevars(db, typevars); + } + if let Some(ty) = self.setter(db) { + ty.find_legacy_typevars(db, typevars); + } + } } bitflags! { @@ -923,6 +936,7 @@ impl<'db> Type<'db> { typevar.definition(db), Some(TypeVarBoundOrConstraints::UpperBound(bound.normalized(db))), typevar.default_ty(db), + typevar.kind(db), )) } Some(TypeVarBoundOrConstraints::Constraints(union)) => { @@ -932,6 +946,7 @@ impl<'db> Type<'db> { typevar.definition(db), Some(TypeVarBoundOrConstraints::Constraints(union.normalized(db))), typevar.default_ty(db), + typevar.kind(db), )) } None => self, @@ -3799,6 +3814,56 @@ impl<'db> Type<'db> { Signatures::single(signature) } + Some(KnownClass::TypeVar) => { + // ```py + // class TypeVar: + // def __new__( + // cls, + // name: str, + // *constraints: Any, + // bound: Any | None = None, + // contravariant: bool = False, + // covariant: bool = False, + // infer_variance: bool = False, + // default: Any = ..., + // ) -> Self: ... + // ``` + let signature = CallableSignature::single( + self, + Signature::new( + Parameters::new([ + Parameter::positional_or_keyword(Name::new_static("name")) + .with_annotated_type(Type::LiteralString), + Parameter::variadic(Name::new_static("constraints")) + .type_form() + .with_annotated_type(Type::any()), + Parameter::keyword_only(Name::new_static("bound")) + .type_form() + .with_annotated_type(UnionType::from_elements( + db, + [Type::any(), Type::none(db)], + )) + .with_default_type(Type::none(db)), + Parameter::keyword_only(Name::new_static("default")) + .type_form() + .with_annotated_type(Type::any()) + .with_default_type(KnownClass::NoneType.to_instance(db)), + Parameter::keyword_only(Name::new_static("contravariant")) + .with_annotated_type(KnownClass::Bool.to_instance(db)) + .with_default_type(Type::BooleanLiteral(false)), + Parameter::keyword_only(Name::new_static("covariant")) + .with_annotated_type(KnownClass::Bool.to_instance(db)) + .with_default_type(Type::BooleanLiteral(false)), + Parameter::keyword_only(Name::new_static("infer_variance")) + .with_annotated_type(KnownClass::Bool.to_instance(db)) + .with_default_type(Type::BooleanLiteral(false)), + ]), + Some(KnownClass::TypeVar.to_instance(db)), + ), + ); + Signatures::single(signature) + } + Some(KnownClass::Property) => { let getter_signature = Signature::new( Parameters::new([ @@ -4834,6 +4899,93 @@ impl<'db> Type<'db> { } } + /// Locates any legacy `TypeVar`s in this type, and adds them to a set. This is used to build + /// up a generic context from any legacy `TypeVar`s that appear in a function parameter list or + /// `Generic` specialization. + pub(crate) fn find_legacy_typevars( + self, + db: &'db dyn Db, + typevars: &mut FxOrderSet>, + ) { + match self { + Type::TypeVar(typevar) => { + if typevar.is_legacy(db) { + typevars.insert(typevar); + } + } + + Type::FunctionLiteral(function) => function.find_legacy_typevars(db, typevars), + + Type::BoundMethod(method) => { + method.self_instance(db).find_legacy_typevars(db, typevars); + method.function(db).find_legacy_typevars(db, typevars); + } + + Type::MethodWrapper( + MethodWrapperKind::FunctionTypeDunderGet(function) + | MethodWrapperKind::FunctionTypeDunderCall(function), + ) => { + function.find_legacy_typevars(db, typevars); + } + + Type::MethodWrapper( + MethodWrapperKind::PropertyDunderGet(property) + | MethodWrapperKind::PropertyDunderSet(property), + ) => { + property.find_legacy_typevars(db, typevars); + } + + Type::Callable(callable) => { + callable.find_legacy_typevars(db, typevars); + } + + Type::PropertyInstance(property) => { + property.find_legacy_typevars(db, typevars); + } + + Type::Union(union) => { + for element in union.iter(db) { + element.find_legacy_typevars(db, typevars); + } + } + Type::Intersection(intersection) => { + for positive in intersection.positive(db) { + positive.find_legacy_typevars(db, typevars); + } + for negative in intersection.negative(db) { + negative.find_legacy_typevars(db, typevars); + } + } + Type::Tuple(tuple) => { + for element in tuple.iter(db) { + element.find_legacy_typevars(db, typevars); + } + } + + Type::Dynamic(_) + | Type::Never + | Type::AlwaysTruthy + | Type::AlwaysFalsy + | Type::WrapperDescriptor(_) + | Type::MethodWrapper(MethodWrapperKind::StrStartswith(_)) + | Type::DataclassDecorator(_) + | Type::DataclassTransformer(_) + | Type::ModuleLiteral(_) + | Type::ClassLiteral(_) + | Type::GenericAlias(_) + | Type::SubclassOf(_) + | Type::IntLiteral(_) + | Type::BooleanLiteral(_) + | Type::LiteralString + | Type::StringLiteral(_) + | Type::BytesLiteral(_) + | Type::SliceLiteral(_) + | Type::BoundSuper(_) + | Type::Instance(_) + | Type::KnownInstance(_) => {} + } + } + /// Return the string representation of this type when converted to string as it would be /// provided by the `__str__` method. /// @@ -4844,9 +4996,7 @@ impl<'db> Type<'db> { match self { Type::IntLiteral(_) | Type::BooleanLiteral(_) => self.repr(db), Type::StringLiteral(_) | Type::LiteralString => *self, - Type::KnownInstance(known_instance) => { - Type::string_literal(db, known_instance.repr(db)) - } + Type::KnownInstance(known_instance) => Type::string_literal(db, known_instance.repr()), // TODO: handle more complex types _ => KnownClass::Str.to_instance(db), } @@ -4864,9 +5014,7 @@ impl<'db> Type<'db> { Type::string_literal(db, &format!("'{}'", literal.value(db).escape_default())) } Type::LiteralString => Type::LiteralString, - Type::KnownInstance(known_instance) => { - Type::string_literal(db, known_instance.repr(db)) - } + Type::KnownInstance(known_instance) => Type::string_literal(db, known_instance.repr()), // TODO: handle more complex types _ => KnownClass::Str.to_instance(db), } @@ -5235,12 +5383,12 @@ impl<'db> InvalidTypeExpression<'db> { InvalidTypeExpression::TypeQualifier(qualifier) => write!( f, "Type qualifier `{q}` is not allowed in type expressions (only in annotation expressions)", - q = qualifier.repr(self.db) + q = qualifier.repr() ), InvalidTypeExpression::TypeQualifierRequiresOneArgument(qualifier) => write!( f, "Type qualifier `{q}` is not allowed in type expressions (only in annotation expressions, and only with exactly one argument)", - q = qualifier.repr(self.db) + q = qualifier.repr() ), InvalidTypeExpression::InvalidType(ty) => write!( f, @@ -5255,6 +5403,13 @@ impl<'db> InvalidTypeExpression<'db> { } } +/// Whether this typecar was created via the legacy `TypeVar` constructor, or using PEP 695 syntax. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum TypeVarKind { + Legacy, + Pep695, +} + /// Data regarding a single type variable. /// /// This is referenced by `KnownInstanceType::TypeVar` (to represent the singleton type of the @@ -5276,9 +5431,15 @@ pub struct TypeVarInstance<'db> { /// The default type for this TypeVar default_ty: Option>, + + pub kind: TypeVarKind, } impl<'db> TypeVarInstance<'db> { + pub(crate) fn is_legacy(self, db: &'db dyn Db) -> bool { + matches!(self.kind(db), TypeVarKind::Legacy) + } + #[allow(unused)] pub(crate) fn upper_bound(self, db: &'db dyn Db) -> Option> { if let Some(TypeVarBoundOrConstraints::UpperBound(ty)) = self.bound_or_constraints(db) { @@ -6368,6 +6529,17 @@ impl<'db> FunctionType<'db> { ) } + fn find_legacy_typevars( + self, + db: &'db dyn Db, + typevars: &mut FxOrderSet>, + ) { + let signatures = self.signature(db); + for signature in signatures { + signature.find_legacy_typevars(db, typevars); + } + } + /// Returns `self` as [`OverloadedFunction`] if it is overloaded, [`None`] otherwise. /// /// ## Note @@ -6698,6 +6870,16 @@ impl<'db> CallableType<'db> { ) } + fn find_legacy_typevars( + self, + db: &'db dyn Db, + typevars: &mut FxOrderSet>, + ) { + for signature in self.signatures(db) { + signature.find_legacy_typevars(db, typevars); + } + } + /// Check whether this callable type is fully static. /// /// See [`Type::is_fully_static`] for more details. diff --git a/crates/red_knot_python_semantic/src/types/call/bind.rs b/crates/red_knot_python_semantic/src/types/call/bind.rs index 8f5fb66658..14e6039c31 100644 --- a/crates/red_knot_python_semantic/src/types/call/bind.rs +++ b/crates/red_knot_python_semantic/src/types/call/bind.rs @@ -16,7 +16,7 @@ use crate::types::diagnostic::{ NO_MATCHING_OVERLOAD, PARAMETER_ALREADY_ASSIGNED, TOO_MANY_POSITIONAL_ARGUMENTS, UNKNOWN_ARGUMENT, }; -use crate::types::generics::{Specialization, SpecializationBuilder}; +use crate::types::generics::{Specialization, SpecializationBuilder, SpecializationError}; use crate::types::signatures::{Parameter, ParameterForm}; use crate::types::{ todo_type, BoundMethodType, DataclassParams, DataclassTransformerParams, FunctionDecorators, @@ -295,54 +295,76 @@ impl<'db> Bindings<'db> { } } - Type::WrapperDescriptor(WrapperDescriptorKind::PropertyDunderGet) => { - match overload.parameter_types() { - [Some(property @ Type::PropertyInstance(_)), Some(instance), ..] - if instance.is_none(db) => + Type::WrapperDescriptor(WrapperDescriptorKind::PropertyDunderGet) => match overload + .parameter_types() + { + [Some(property @ Type::PropertyInstance(_)), Some(instance), ..] + if instance.is_none(db) => + { + overload.set_return_type(*property); + } + [Some(Type::PropertyInstance(property)), Some(Type::KnownInstance(KnownInstanceType::TypeAliasType(type_alias))), ..] + if property.getter(db).is_some_and(|getter| { + getter + .into_function_literal() + .is_some_and(|f| f.name(db) == "__name__") + }) => + { + overload.set_return_type(Type::string_literal(db, type_alias.name(db))); + } + [Some(Type::PropertyInstance(property)), Some(Type::KnownInstance(KnownInstanceType::TypeVar(typevar))), ..] => { + match property + .getter(db) + .and_then(Type::into_function_literal) + .map(|f| f.name(db).as_str()) { - overload.set_return_type(*property); + Some("__name__") => { + overload + .set_return_type(Type::string_literal(db, typevar.name(db))); + } + Some("__bound__") => { + overload.set_return_type( + typevar.upper_bound(db).unwrap_or_else(|| Type::none(db)), + ); + } + Some("__constraints__") => { + overload.set_return_type(TupleType::from_elements( + db, + typevar.constraints(db).into_iter().flatten(), + )); + } + Some("__default__") => { + overload.set_return_type( + typevar.default_ty(db).unwrap_or_else(|| { + KnownClass::NoDefaultType.to_instance(db) + }), + ); + } + _ => {} } - [Some(Type::PropertyInstance(property)), Some(Type::KnownInstance(KnownInstanceType::TypeAliasType(type_alias))), ..] - if property.getter(db).is_some_and(|getter| { - getter - .into_function_literal() - .is_some_and(|f| f.name(db) == "__name__") - }) => - { - overload.set_return_type(Type::string_literal(db, type_alias.name(db))); - } - [Some(Type::PropertyInstance(property)), Some(Type::KnownInstance(KnownInstanceType::TypeVar(type_var))), ..] - if property.getter(db).is_some_and(|getter| { - getter - .into_function_literal() - .is_some_and(|f| f.name(db) == "__name__") - }) => - { - overload.set_return_type(Type::string_literal(db, type_var.name(db))); - } - [Some(Type::PropertyInstance(property)), Some(instance), ..] => { - if let Some(getter) = property.getter(db) { - if let Ok(return_ty) = getter - .try_call(db, CallArgumentTypes::positional([*instance])) - .map(|binding| binding.return_type(db)) - { - overload.set_return_type(return_ty); - } else { - overload.errors.push(BindingError::InternalCallError( - "calling the getter failed", - )); - overload.set_return_type(Type::unknown()); - } + } + [Some(Type::PropertyInstance(property)), Some(instance), ..] => { + if let Some(getter) = property.getter(db) { + if let Ok(return_ty) = getter + .try_call(db, CallArgumentTypes::positional([*instance])) + .map(|binding| binding.return_type(db)) + { + overload.set_return_type(return_ty); } else { overload.errors.push(BindingError::InternalCallError( - "property has no getter", + "calling the getter failed", )); - overload.set_return_type(Type::Never); + overload.set_return_type(Type::unknown()); } + } else { + overload + .errors + .push(BindingError::InternalCallError("property has no getter")); + overload.set_return_type(Type::Never); } - _ => {} } - } + _ => {} + }, Type::MethodWrapper(MethodWrapperKind::PropertyDunderGet(property)) => { match overload.parameter_types() { @@ -1150,29 +1172,6 @@ impl<'db> Binding<'db> { signature: &Signature<'db>, argument_types: &CallArgumentTypes<'_, 'db>, ) { - // If this overload is generic, first see if we can infer a specialization of the function - // from the arguments that were passed in. - let parameters = signature.parameters(); - if signature.generic_context.is_some() || signature.inherited_generic_context.is_some() { - let mut builder = SpecializationBuilder::new(db); - for (argument_index, (_, argument_type)) in argument_types.iter().enumerate() { - let Some(parameter_index) = self.argument_parameters[argument_index] else { - // There was an error with argument when matching parameters, so don't bother - // type-checking it. - continue; - }; - let parameter = ¶meters[parameter_index]; - let Some(expected_type) = parameter.annotated_type() else { - continue; - }; - builder.infer(expected_type, argument_type); - } - self.specialization = signature.generic_context.map(|gc| builder.build(gc)); - self.inherited_specialization = signature - .inherited_generic_context - .map(|gc| builder.build(gc)); - } - let mut num_synthetic_args = 0; let get_argument_index = |argument_index: usize, num_synthetic_args: usize| { if argument_index >= num_synthetic_args { @@ -1185,6 +1184,39 @@ impl<'db> Binding<'db> { None } }; + + // If this overload is generic, first see if we can infer a specialization of the function + // from the arguments that were passed in. + let parameters = signature.parameters(); + if signature.generic_context.is_some() || signature.inherited_generic_context.is_some() { + let mut builder = SpecializationBuilder::new(db); + for (argument_index, (argument, argument_type)) in argument_types.iter().enumerate() { + if matches!(argument, Argument::Synthetic) { + num_synthetic_args += 1; + } + let Some(parameter_index) = self.argument_parameters[argument_index] else { + // There was an error with argument when matching parameters, so don't bother + // type-checking it. + continue; + }; + let parameter = ¶meters[parameter_index]; + let Some(expected_type) = parameter.annotated_type() else { + continue; + }; + if let Err(error) = builder.infer(expected_type, argument_type) { + self.errors.push(BindingError::SpecializationError { + error, + argument_index: get_argument_index(argument_index, num_synthetic_args), + }); + } + } + self.specialization = signature.generic_context.map(|gc| builder.build(gc)); + self.inherited_specialization = signature + .inherited_generic_context + .map(|gc| builder.build(gc)); + } + + num_synthetic_args = 0; for (argument_index, (argument, argument_type)) in argument_types.iter().enumerate() { if matches!(argument, Argument::Synthetic) { num_synthetic_args += 1; @@ -1250,6 +1282,20 @@ impl<'db> Binding<'db> { &self.parameter_tys } + pub(crate) fn arguments_for_parameter<'a>( + &'a self, + argument_types: &'a CallArgumentTypes<'a, 'db>, + parameter_index: usize, + ) -> impl Iterator, Type<'db>)> + 'a { + argument_types + .iter() + .zip(&self.argument_parameters) + .filter(move |(_, argument_parameter)| { + argument_parameter.is_some_and(|ap| ap == parameter_index) + }) + .map(|(arg_and_type, _)| arg_and_type) + } + fn report_diagnostics( &self, context: &InferContext<'db>, @@ -1398,6 +1444,11 @@ pub(crate) enum BindingError<'db> { argument_index: Option, parameter: ParameterContext, }, + /// An inferred specialization was invalid. + SpecializationError { + error: SpecializationError<'db>, + argument_index: Option, + }, /// The call itself might be well constructed, but an error occurred while evaluating the call. /// We use this variant to report errors in `property.__get__` and `property.__set__`, which /// can occur when the call to the underlying getter/setter fails. @@ -1510,6 +1561,35 @@ impl<'db> BindingError<'db> { } } + Self::SpecializationError { + error, + argument_index, + } => { + let range = Self::get_node(node, *argument_index); + let Some(builder) = context.report_lint(&INVALID_ARGUMENT_TYPE, range) else { + return; + }; + + let typevar = error.typevar(); + let argument_type = error.argument_type(); + let argument_ty_display = argument_type.display(context.db()); + + let mut diag = builder.into_diagnostic("Argument to this function is incorrect"); + diag.set_primary_message(format_args!( + "Argument type `{argument_ty_display}` does not satisfy {} of type variable `{}`", + match error { + SpecializationError::MismatchedBound {..} => "upper bound", + SpecializationError::MismatchedConstraint {..} => "constraints", + }, + typevar.name(context.db()), + )); + + let typevar_range = typevar.definition(context.db()).full_range(context.db()); + let mut sub = SubDiagnostic::new(Severity::Info, "Type variable defined here"); + sub.annotate(Annotation::primary(typevar_range.into())); + diag.sub(sub); + } + Self::InternalCallError(reason) => { let node = Self::get_node(node, None); if let Some(builder) = context.report_lint(&CALL_NON_CALLABLE, node) { diff --git a/crates/red_knot_python_semantic/src/types/diagnostic.rs b/crates/red_knot_python_semantic/src/types/diagnostic.rs index e051de715b..63d5a18e4b 100644 --- a/crates/red_knot_python_semantic/src/types/diagnostic.rs +++ b/crates/red_knot_python_semantic/src/types/diagnostic.rs @@ -35,6 +35,7 @@ pub(crate) fn register_lints(registry: &mut LintRegistryBuilder) { registry.register_lint(&INVALID_CONTEXT_MANAGER); registry.register_lint(&INVALID_DECLARATION); registry.register_lint(&INVALID_EXCEPTION_CAUGHT); + registry.register_lint(&INVALID_LEGACY_TYPE_VARIABLE); registry.register_lint(&INVALID_METACLASS); registry.register_lint(&INVALID_PARAMETER_DEFAULT); registry.register_lint(&INVALID_PROTOCOL); @@ -391,6 +392,34 @@ declare_lint! { } } +declare_lint! { + /// ## What it does + /// Checks for the creation of invalid legacy `TypeVar`s + /// + /// ## Why is this bad? + /// There are several requirements that you must follow when creating a legacy `TypeVar`. + /// + /// ## Examples + /// ```python + /// from typing import TypeVar + /// + /// T = TypeVar("T") # okay + /// Q = TypeVar("S") # error: TypeVar name must match the variable it's assigned to + /// T = TypeVar("T") # error: TypeVars should not be redefined + /// + /// # error: TypeVar must be immediately assigned to a variable + /// def f(t: TypeVar("U")): ... + /// ``` + /// + /// ## References + /// - [Typing spec: Generics](https://typing.python.org/en/latest/spec/generics.html#introduction) + pub(crate) static INVALID_LEGACY_TYPE_VARIABLE = { + summary: "detects invalid legacy type variables", + status: LintStatus::preview("1.0.0"), + default_level: Level::Error, + } +} + declare_lint! { /// ## What it does /// Checks for arguments to `metaclass=` that are invalid. @@ -1314,7 +1343,7 @@ pub(crate) fn report_invalid_arguments_to_annotated( builder.into_diagnostic(format_args!( "Special form `{}` expected at least 2 arguments \ (one type and at least one metadata element)", - KnownInstanceType::Annotated.repr(context.db()) + KnownInstanceType::Annotated.repr() )); } @@ -1362,7 +1391,7 @@ pub(crate) fn report_invalid_arguments_to_callable( }; builder.into_diagnostic(format_args!( "Special form `{}` expected exactly two arguments (parameter types and return type)", - KnownInstanceType::Callable.repr(context.db()) + KnownInstanceType::Callable.repr() )); } diff --git a/crates/red_knot_python_semantic/src/types/display.rs b/crates/red_knot_python_semantic/src/types/display.rs index 4220b0cf41..2633938ae9 100644 --- a/crates/red_knot_python_semantic/src/types/display.rs +++ b/crates/red_knot_python_semantic/src/types/display.rs @@ -94,7 +94,7 @@ impl Display for DisplayRepresentation<'_> { SubclassOfInner::Class(class) => write!(f, "type[{}]", class.name(self.db)), SubclassOfInner::Dynamic(dynamic) => write!(f, "type[{dynamic}]"), }, - Type::KnownInstance(known_instance) => f.write_str(known_instance.repr(self.db)), + Type::KnownInstance(known_instance) => f.write_str(known_instance.repr()), Type::FunctionLiteral(function) => { let signature = function.signature(self.db); diff --git a/crates/red_knot_python_semantic/src/types/generics.rs b/crates/red_knot_python_semantic/src/types/generics.rs index b1c0355ebf..8589f14a5c 100644 --- a/crates/red_knot_python_semantic/src/types/generics.rs +++ b/crates/red_knot_python_semantic/src/types/generics.rs @@ -5,9 +5,9 @@ use crate::semantic_index::SemanticIndex; use crate::types::signatures::{Parameter, Parameters, Signature}; use crate::types::{ declaration_type, KnownInstanceType, Type, TypeVarBoundOrConstraints, TypeVarInstance, - UnionBuilder, UnionType, + UnionType, }; -use crate::Db; +use crate::{Db, FxOrderSet}; /// A list of formal type variables for a generic function, class, or type alias. /// @@ -20,6 +20,7 @@ pub struct GenericContext<'db> { } impl<'db> GenericContext<'db> { + /// Creates a generic context from a list of PEP-695 type parameters. pub(crate) fn from_type_params( db: &'db dyn Db, index: &'db SemanticIndex<'db>, @@ -53,6 +54,32 @@ impl<'db> GenericContext<'db> { } } + /// Creates a generic context from the legecy `TypeVar`s that appear in a function parameter + /// list. + pub(crate) fn from_function_params( + db: &'db dyn Db, + parameters: &Parameters<'db>, + return_type: Option>, + ) -> Option { + let mut variables = FxOrderSet::default(); + for param in parameters { + if let Some(ty) = param.annotated_type() { + ty.find_legacy_typevars(db, &mut variables); + } + if let Some(ty) = param.default_type() { + ty.find_legacy_typevars(db, &mut variables); + } + } + if let Some(ty) = return_type { + ty.find_legacy_typevars(db, &mut variables); + } + if variables.is_empty() { + return None; + } + let variables: Box<[_]> = variables.into_iter().collect(); + Some(Self::new(db, variables)) + } + pub(crate) fn signature(self, db: &'db dyn Db) -> Signature<'db> { let parameters = Parameters::new( self.variables(db) @@ -303,7 +330,7 @@ impl<'db> Specialization<'db> { /// specialization of a generic function. pub(crate) struct SpecializationBuilder<'db> { db: &'db dyn Db, - types: FxHashMap, UnionBuilder<'db>>, + types: FxHashMap, Type<'db>>, } impl<'db> SpecializationBuilder<'db> { @@ -320,8 +347,8 @@ impl<'db> SpecializationBuilder<'db> { .iter() .map(|variable| { self.types - .remove(variable) - .map(UnionBuilder::build) + .get(variable) + .copied() .unwrap_or(variable.default_ty(self.db).unwrap_or(Type::unknown())) }) .collect(); @@ -329,17 +356,25 @@ impl<'db> SpecializationBuilder<'db> { } fn add_type_mapping(&mut self, typevar: TypeVarInstance<'db>, ty: Type<'db>) { - let builder = self - .types + self.types .entry(typevar) - .or_insert_with(|| UnionBuilder::new(self.db)); - builder.add_in_place(ty); + .and_modify(|existing| { + *existing = UnionType::from_elements(self.db, [*existing, ty]); + }) + .or_insert(ty); } - pub(crate) fn infer(&mut self, formal: Type<'db>, actual: Type<'db>) { - // If the actual type is already assignable to the formal type, then return without adding - // any new type mappings. (Note that if the formal type contains any typevars, this check - // will fail, since no non-typevar types are assignable to a typevar.) + pub(crate) fn infer( + &mut self, + formal: Type<'db>, + actual: Type<'db>, + ) -> Result<(), SpecializationError<'db>> { + // If the actual type is a subtype of the formal type, then return without adding any new + // type mappings. (Note that if the formal type contains any typevars, this check will + // fail, since no non-typevar types are assignable to a typevar. Also note that we are + // checking _subtyping_, not _assignability_, so that we do specialize typevars to dynamic + // argument types; and we have a special case for `Never`, which is a subtype of all types, + // but which we also do want as a specialization candidate.) // // In particular, this handles a case like // @@ -350,12 +385,37 @@ impl<'db> SpecializationBuilder<'db> { // ``` // // without specializing `T` to `None`. - if actual.is_assignable_to(self.db, formal) { - return; + if !actual.is_never() && actual.is_subtype_of(self.db, formal) { + return Ok(()); } match (formal, actual) { - (Type::TypeVar(typevar), _) => self.add_type_mapping(typevar, actual), + (Type::TypeVar(typevar), _) => match typevar.bound_or_constraints(self.db) { + Some(TypeVarBoundOrConstraints::UpperBound(bound)) => { + if !actual.is_assignable_to(self.db, bound) { + return Err(SpecializationError::MismatchedBound { + typevar, + argument: actual, + }); + } + self.add_type_mapping(typevar, actual); + } + Some(TypeVarBoundOrConstraints::Constraints(constraints)) => { + for constraint in constraints.iter(self.db) { + if actual.is_assignable_to(self.db, *constraint) { + self.add_type_mapping(typevar, *constraint); + return Ok(()); + } + } + return Err(SpecializationError::MismatchedConstraint { + typevar, + argument: actual, + }); + } + _ => { + self.add_type_mapping(typevar, actual); + } + }, (Type::Tuple(formal_tuple), Type::Tuple(actual_tuple)) => { let formal_elements = formal_tuple.elements(self.db); @@ -364,7 +424,7 @@ impl<'db> SpecializationBuilder<'db> { for (formal_element, actual_element) in formal_elements.iter().zip(actual_elements) { - self.infer(*formal_element, *actual_element); + self.infer(*formal_element, *actual_element)?; } } } @@ -397,12 +457,42 @@ impl<'db> SpecializationBuilder<'db> { // actual type must also be disjoint from every negative element of the // intersection, but that doesn't help us infer any type mappings.) for positive in formal.iter_positive(self.db) { - self.infer(positive, actual); + self.infer(positive, actual)?; } } // TODO: Add more forms that we can structurally induct into: type[C], callables _ => {} } + + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum SpecializationError<'db> { + MismatchedBound { + typevar: TypeVarInstance<'db>, + argument: Type<'db>, + }, + MismatchedConstraint { + typevar: TypeVarInstance<'db>, + argument: Type<'db>, + }, +} + +impl<'db> SpecializationError<'db> { + pub(crate) fn typevar(&self) -> TypeVarInstance<'db> { + match self { + Self::MismatchedBound { typevar, .. } => *typevar, + Self::MismatchedConstraint { typevar, .. } => *typevar, + } + } + + pub(crate) fn argument_type(&self) -> Type<'db> { + match self { + Self::MismatchedBound { argument, .. } => *argument, + Self::MismatchedConstraint { argument, .. } => *argument, + } } } diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index e9448d83b0..db5b2a5319 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -73,9 +73,9 @@ use crate::types::diagnostic::{ CALL_POSSIBLY_UNBOUND_METHOD, CONFLICTING_DECLARATIONS, CONFLICTING_METACLASS, CYCLIC_CLASS_DEFINITION, DIVISION_BY_ZERO, DUPLICATE_BASE, INCONSISTENT_MRO, INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, INVALID_BASE, INVALID_DECLARATION, - INVALID_PARAMETER_DEFAULT, INVALID_TYPE_FORM, INVALID_TYPE_VARIABLE_CONSTRAINTS, - POSSIBLY_UNBOUND_IMPORT, UNDEFINED_REVEAL, UNRESOLVED_ATTRIBUTE, UNRESOLVED_IMPORT, - UNSUPPORTED_OPERATOR, + INVALID_LEGACY_TYPE_VARIABLE, INVALID_PARAMETER_DEFAULT, INVALID_TYPE_FORM, + INVALID_TYPE_VARIABLE_CONSTRAINTS, POSSIBLY_UNBOUND_IMPORT, UNDEFINED_REVEAL, + UNRESOLVED_ATTRIBUTE, UNRESOLVED_IMPORT, UNSUPPORTED_OPERATOR, }; use crate::types::generics::GenericContext; use crate::types::mro::MroErrorKind; @@ -87,7 +87,8 @@ use crate::types::{ MemberLookupPolicy, MetaclassCandidate, Parameter, ParameterForm, Parameters, Signature, Signatures, SliceLiteralType, StringLiteralType, SubclassOfType, Symbol, SymbolAndQualifiers, Truthiness, TupleType, Type, TypeAliasType, TypeAndQualifiers, TypeArrayDisplay, - TypeQualifiers, TypeVarBoundOrConstraints, TypeVarInstance, UnionBuilder, UnionType, + TypeQualifiers, TypeVarBoundOrConstraints, TypeVarInstance, TypeVarKind, UnionBuilder, + UnionType, }; use crate::unpack::{Unpack, UnpackPosition}; use crate::util::subscript::{PyIndex, PySlice}; @@ -2204,6 +2205,7 @@ impl<'db> TypeInferenceBuilder<'db> { definition, bound_or_constraint, default_ty, + TypeVarKind::Pep695, ))); self.add_declaration_with_binding( node.into(), @@ -3733,7 +3735,9 @@ impl<'db> TypeInferenceBuilder<'db> { ast::Expr::Named(named) => self.infer_named_expression(named), ast::Expr::If(if_expression) => self.infer_if_expression(if_expression), ast::Expr::Lambda(lambda_expression) => self.infer_lambda_expression(lambda_expression), - ast::Expr::Call(call_expression) => self.infer_call_expression(call_expression), + ast::Expr::Call(call_expression) => { + self.infer_call_expression(expression, call_expression) + } ast::Expr::Starred(starred) => self.infer_starred_expression(starred), ast::Expr::Yield(yield_expression) => self.infer_yield_expression(yield_expression), ast::Expr::YieldFrom(yield_from) => self.infer_yield_from_expression(yield_from), @@ -4277,7 +4281,11 @@ impl<'db> TypeInferenceBuilder<'db> { }) } - fn infer_call_expression(&mut self, call_expression: &ast::ExprCall) -> Type<'db> { + fn infer_call_expression( + &mut self, + call_expression_node: &ast::Expr, + call_expression: &ast::ExprCall, + ) -> Type<'db> { let ast::ExprCall { range: _, func, @@ -4332,6 +4340,7 @@ impl<'db> TypeInferenceBuilder<'db> { | KnownClass::Object | KnownClass::Property | KnownClass::Super + | KnownClass::TypeVar ) ) { @@ -4543,70 +4552,179 @@ impl<'db> TypeInferenceBuilder<'db> { _ => {} } } - Type::ClassLiteral(class) - if class.is_known(self.db(), KnownClass::Super) => - { - // Handle the case where `super()` is called with no arguments. - // In this case, we need to infer the two arguments: - // 1. The nearest enclosing class - // 2. The first parameter of the current function (typically `self` or `cls`) - match overload.parameter_types() { - [] => { - let scope = self.scope(); - let Some(enclosing_class) = self.enclosing_class_symbol(scope) + Type::ClassLiteral(class) => { + let Some(known_class) = class.known(self.db()) else { + continue; + }; + + match known_class { + KnownClass::Super => { + // Handle the case where `super()` is called with no arguments. + // In this case, we need to infer the two arguments: + // 1. The nearest enclosing class + // 2. The first parameter of the current function (typically `self` or `cls`) + match overload.parameter_types() { + [] => { + let scope = self.scope(); + + let Some(enclosing_class) = + self.enclosing_class_symbol(scope) + else { + overload.set_return_type(Type::unknown()); + BoundSuperError::UnavailableImplicitArguments + .report_diagnostic( + &self.context, + call_expression.into(), + ); + continue; + }; + + let Some(first_param) = + self.first_param_type_in_scope(scope) + else { + overload.set_return_type(Type::unknown()); + BoundSuperError::UnavailableImplicitArguments + .report_diagnostic( + &self.context, + call_expression.into(), + ); + continue; + }; + + let bound_super = BoundSuperType::build( + self.db(), + enclosing_class, + first_param, + ) + .unwrap_or_else(|err| { + err.report_diagnostic( + &self.context, + call_expression.into(), + ); + Type::unknown() + }); + + overload.set_return_type(bound_super); + } + [Some(pivot_class_type), Some(owner_type)] => { + let bound_super = BoundSuperType::build( + self.db(), + *pivot_class_type, + *owner_type, + ) + .unwrap_or_else(|err| { + err.report_diagnostic( + &self.context, + call_expression.into(), + ); + Type::unknown() + }); + + overload.set_return_type(bound_super); + } + _ => (), + } + } + + KnownClass::TypeVar => { + let assigned_to = (self.index) + .try_expression(call_expression_node) + .and_then(|expr| expr.assigned_to(self.db())); + + let Some(target) = + assigned_to.as_ref().and_then(|assigned_to| { + match assigned_to.node().targets.as_slice() { + [ast::Expr::Name(target)] => Some(target), + _ => None, + } + }) else { - overload.set_return_type(Type::unknown()); - BoundSuperError::UnavailableImplicitArguments - .report_diagnostic( - &self.context, - call_expression.into(), - ); + if let Some(builder) = self.context.report_lint( + &INVALID_LEGACY_TYPE_VARIABLE, + call_expression, + ) { + builder.into_diagnostic(format_args!( + "A legacy `typing.TypeVar` must be immediately assigned to a variable", + )); + } continue; }; - let Some(first_param) = self.first_param_type_in_scope(scope) + let [Some(name_param), constraints, bound, default, _contravariant, _covariant, _infer_variance] = + overload.parameter_types() else { - overload.set_return_type(Type::unknown()); - BoundSuperError::UnavailableImplicitArguments - .report_diagnostic( - &self.context, - call_expression.into(), - ); continue; }; - let bound_super = BoundSuperType::build( - self.db(), - enclosing_class, - first_param, - ) - .unwrap_or_else(|err| { - err.report_diagnostic( - &self.context, - call_expression.into(), - ); - Type::unknown() - }); + let name_param = name_param + .into_string_literal() + .map(|name| name.value(self.db()).as_ref()); + if name_param.is_none_or(|name_param| name_param != target.id) { + if let Some(builder) = self.context.report_lint( + &INVALID_LEGACY_TYPE_VARIABLE, + call_expression, + ) { + builder.into_diagnostic(format_args!( + "The name of a legacy `typing.TypeVar`{} must match \ + the name of the variable it is assigned to (`{}`)", + if let Some(name_param) = name_param { + format!(" (`{name_param}`)") + } else { + String::new() + }, + target.id, + )); + } + continue; + } - overload.set_return_type(bound_super); - } - [Some(pivot_class_type), Some(owner_type)] => { - let bound_super = BoundSuperType::build( - self.db(), - *pivot_class_type, - *owner_type, - ) - .unwrap_or_else(|err| { - err.report_diagnostic( - &self.context, - call_expression.into(), - ); - Type::unknown() - }); + let bound_or_constraint = match (bound, constraints) { + (Some(bound), None) => { + Some(TypeVarBoundOrConstraints::UpperBound(*bound)) + } - overload.set_return_type(bound_super); + (None, Some(_constraints)) => { + // We don't use UnionType::from_elements or UnionBuilder here, + // because we don't want to simplify the list of constraints like + // we do with the elements of an actual union type. + // TODO: Consider using a new `OneOfType` connective here instead, + // since that more accurately represents the actual semantics of + // typevar constraints. + let elements = UnionType::new( + self.db(), + overload + .arguments_for_parameter( + &call_argument_types, + 1, + ) + .map(|(_, ty)| ty) + .collect::>(), + ); + Some(TypeVarBoundOrConstraints::Constraints(elements)) + } + + // TODO: Emit a diagnostic that TypeVar cannot be both bounded and + // constrained + (Some(_), Some(_)) => continue, + + (None, None) => None, + }; + + let containing_assignment = + self.index.expect_single_definition(target); + overload.set_return_type(Type::KnownInstance( + KnownInstanceType::TypeVar(TypeVarInstance::new( + self.db(), + target.id.clone(), + containing_assignment, + bound_or_constraint, + *default, + TypeVarKind::Legacy, + )), + )); } + _ => (), } } @@ -6509,7 +6627,12 @@ impl<'db> TypeInferenceBuilder<'db> { if class.generic_context(self.db()).is_some() { // TODO: specialize the generic class using these explicit type - // variable assignments + // variable assignments. This branch is only encountered when an + // explicit class specialization appears inside of some other subscript + // expression, e.g. `tuple[list[int], ...]`. We have already inferred + // the type of the outer subscript slice as a value expression, which + // means we can't re-infer the inner specialization here as a type + // expression. return value_ty; } } @@ -6753,7 +6876,7 @@ impl<'db> TypeInferenceBuilder<'db> { builder.into_diagnostic(format_args!( "Type qualifier `{type_qualifier}` \ expects exactly one type parameter", - type_qualifier = known_instance.repr(self.db()), + type_qualifier = known_instance.repr(), )); } Type::unknown().into() @@ -7111,7 +7234,7 @@ impl<'db> TypeInferenceBuilder<'db> { } ast::Expr::Call(call_expr) => { - self.infer_call_expression(call_expr); + self.infer_call_expression(expression, call_expr); self.report_invalid_type_expression( expression, format_args!("Function calls are not allowed in type expressions"), @@ -7531,7 +7654,7 @@ impl<'db> TypeInferenceBuilder<'db> { if let Some(builder) = self.context.report_lint(&INVALID_TYPE_FORM, subscript) { builder.into_diagnostic(format_args!( "Special form `{}` expected exactly one type parameter", - known_instance.repr(db) + known_instance.repr() )); } Type::unknown() @@ -7558,7 +7681,7 @@ impl<'db> TypeInferenceBuilder<'db> { if let Some(builder) = self.context.report_lint(&INVALID_TYPE_FORM, subscript) { builder.into_diagnostic(format_args!( "Special form `{}` expected exactly one type parameter", - known_instance.repr(db) + known_instance.repr() )); } Type::unknown() @@ -7574,7 +7697,7 @@ impl<'db> TypeInferenceBuilder<'db> { if let Some(builder) = self.context.report_lint(&INVALID_TYPE_FORM, subscript) { builder.into_diagnostic(format_args!( "Special form `{}` expected exactly one type parameter", - known_instance.repr(db) + known_instance.repr() )); } Type::unknown() @@ -7607,7 +7730,7 @@ impl<'db> TypeInferenceBuilder<'db> { "Expected the first argument to `{}` \ to be a callable object, \ but got an object of type `{}`", - known_instance.repr(db), + known_instance.repr(), argument_type.display(db) )); } @@ -7672,7 +7795,7 @@ impl<'db> TypeInferenceBuilder<'db> { builder.into_diagnostic(format_args!( "Type qualifier `{}` is not allowed in type expressions \ (only in annotation expressions)", - known_instance.repr(db) + known_instance.repr() )); } self.infer_type_expression(arguments_slice) @@ -7715,7 +7838,7 @@ impl<'db> TypeInferenceBuilder<'db> { if let Some(builder) = self.context.report_lint(&INVALID_TYPE_FORM, subscript) { builder.into_diagnostic(format_args!( "Type `{}` expected no type parameter", - known_instance.repr(db) + known_instance.repr() )); } Type::unknown() @@ -7729,7 +7852,7 @@ impl<'db> TypeInferenceBuilder<'db> { if let Some(builder) = self.context.report_lint(&INVALID_TYPE_FORM, subscript) { builder.into_diagnostic(format_args!( "Special form `{}` expected no type parameter", - known_instance.repr(db) + known_instance.repr() )); } Type::unknown() @@ -7740,7 +7863,7 @@ impl<'db> TypeInferenceBuilder<'db> { if let Some(builder) = self.context.report_lint(&INVALID_TYPE_FORM, subscript) { let mut diag = builder.into_diagnostic(format_args!( "Type `{}` expected no type parameter", - known_instance.repr(db) + known_instance.repr() )); diag.info("Did you mean to use `Literal[...]` instead?"); } @@ -8288,7 +8411,7 @@ mod tests { constraints: Option<&[&'static str]>, default: Option<&'static str>| { let var_ty = get_symbol(&db, "src/a.py", &["f"], var).expect_type(); - assert_eq!(var_ty.display(&db).to_string(), var); + assert_eq!(var_ty.display(&db).to_string(), "typing.TypeVar"); let expected_name_ty = format!(r#"Literal["{var}"]"#); let name_ty = var_ty.member(&db, "__name__").symbol.expect_type(); diff --git a/crates/red_knot_python_semantic/src/types/known_instance.rs b/crates/red_knot_python_semantic/src/types/known_instance.rs index 01316c1c56..d6da1d76d4 100644 --- a/crates/red_knot_python_semantic/src/types/known_instance.rs +++ b/crates/red_knot_python_semantic/src/types/known_instance.rs @@ -109,6 +109,10 @@ impl<'db> KnownInstanceType<'db> { | Self::Literal | Self::LiteralString | Self::Optional + // This is a legacy `TypeVar` _outside_ of any generic class or function, so it's + // AlwaysTrue. The truthiness of a typevar inside of a generic class or function + // depends on its bounds and constraints; but that's represented by `Type::TypeVar` and + // handled in elsewhere. | Self::TypeVar(_) | Self::Union | Self::NoReturn @@ -152,7 +156,7 @@ impl<'db> KnownInstanceType<'db> { } /// Return the repr of the symbol at runtime - pub(crate) fn repr(self, db: &'db dyn Db) -> &'db str { + pub(crate) fn repr(self) -> &'db str { match self { Self::Annotated => "typing.Annotated", Self::Literal => "typing.Literal", @@ -188,7 +192,10 @@ impl<'db> KnownInstanceType<'db> { Self::Protocol => "typing.Protocol", Self::Generic => "typing.Generic", Self::ReadOnly => "typing.ReadOnly", - Self::TypeVar(typevar) => typevar.name(db), + // This is a legacy `TypeVar` _outside_ of any generic class or function, so we render + // it as an instance of `typing.TypeVar`. Inside of a generic class or function, we'll + // have a `Type::TypeVar(_)`, which is rendered as the typevar's name. + Self::TypeVar(_) => "typing.TypeVar", Self::TypeAliasType(_) => "typing.TypeAliasType", Self::Unknown => "knot_extensions.Unknown", Self::AlwaysTruthy => "knot_extensions.AlwaysTruthy", diff --git a/crates/red_knot_python_semantic/src/types/signatures.rs b/crates/red_knot_python_semantic/src/types/signatures.rs index 811965cde9..a44720d38c 100644 --- a/crates/red_knot_python_semantic/src/types/signatures.rs +++ b/crates/red_knot_python_semantic/src/types/signatures.rs @@ -18,8 +18,8 @@ use smallvec::{smallvec, SmallVec}; use super::{definition_expression_type, DynamicType, Type}; use crate::semantic_index::definition::Definition; use crate::types::generics::{GenericContext, Specialization}; -use crate::types::todo_type; -use crate::Db; +use crate::types::{todo_type, TypeVarInstance}; +use crate::{Db, FxOrderSet}; use ruff_python_ast::{self as ast, name::Name}; /// The signature of a possible union of callables. @@ -267,6 +267,8 @@ impl<'db> Signature<'db> { definition: Definition<'db>, function_node: &ast::StmtFunctionDef, ) -> Self { + let parameters = + Parameters::from_parameters(db, definition, function_node.parameters.as_ref()); let return_ty = function_node.returns.as_ref().map(|returns| { if function_node.is_async { todo_type!("generic types.CoroutineType") @@ -274,15 +276,17 @@ impl<'db> Signature<'db> { definition_expression_type(db, definition, returns.as_ref()) } }); + let legacy_generic_context = + GenericContext::from_function_params(db, ¶meters, return_ty); + + if generic_context.is_some() && legacy_generic_context.is_some() { + // TODO: Raise a diagnostic! + } Self { - generic_context, + generic_context: generic_context.or(legacy_generic_context), inherited_generic_context, - parameters: Parameters::from_parameters( - db, - definition, - function_node.parameters.as_ref(), - ), + parameters, return_ty, } } @@ -315,6 +319,24 @@ impl<'db> Signature<'db> { } } + pub(crate) fn find_legacy_typevars( + &self, + db: &'db dyn Db, + typevars: &mut FxOrderSet>, + ) { + for param in &self.parameters { + if let Some(ty) = param.annotated_type() { + ty.find_legacy_typevars(db, typevars); + } + if let Some(ty) = param.default_type() { + ty.find_legacy_typevars(db, typevars); + } + } + if let Some(ty) = self.return_ty { + ty.find_legacy_typevars(db, typevars); + } + } + /// Return the parameters in this signature. pub(crate) fn parameters(&self) -> &Parameters<'db> { &self.parameters diff --git a/crates/ruff_benchmark/benches/red_knot.rs b/crates/ruff_benchmark/benches/red_knot.rs index e0b0da246b..6dab343b05 100644 --- a/crates/ruff_benchmark/benches/red_knot.rs +++ b/crates/ruff_benchmark/benches/red_knot.rs @@ -59,13 +59,22 @@ type KeyDiagnosticFields = ( Severity, ); -static EXPECTED_TOMLLIB_DIAGNOSTICS: &[KeyDiagnosticFields] = &[( - DiagnosticId::lint("unused-ignore-comment"), - Some("/src/tomllib/_parser.py"), - Some(22299..22333), - "Unused blanket `type: ignore` directive", - Severity::Warning, -)]; +static EXPECTED_TOMLLIB_DIAGNOSTICS: &[KeyDiagnosticFields] = &[ + ( + DiagnosticId::lint("no-matching-overload"), + Some("/src/tomllib/_parser.py"), + Some(2329..2358), + "No overload of bound method `__init__` matches arguments", + Severity::Error, + ), + ( + DiagnosticId::lint("unused-ignore-comment"), + Some("/src/tomllib/_parser.py"), + Some(22299..22333), + "Unused blanket `type: ignore` directive", + Severity::Warning, + ), +]; fn tomllib_path(file: &TestFile) -> SystemPathBuf { SystemPathBuf::from("src").join(file.name()) diff --git a/knot.schema.json b/knot.schema.json index 28a09099c6..66e0a5b2e3 100644 --- a/knot.schema.json +++ b/knot.schema.json @@ -450,6 +450,16 @@ } ] }, + "invalid-legacy-type-variable": { + "title": "detects invalid legacy type variables", + "description": "## What it does\nChecks for the creation of invalid legacy `TypeVar`s\n\n## Why is this bad?\nThere are several requirements that you must follow when creating a legacy `TypeVar`.\n\n## Examples\n```python\nfrom typing import TypeVar\n\nT = TypeVar(\"T\") # okay\nQ = TypeVar(\"S\") # error: TypeVar name must match the variable it's assigned to\nT = TypeVar(\"T\") # error: TypeVars should not be redefined\n\n# error: TypeVar must be immediately assigned to a variable\ndef f(t: TypeVar(\"U\")): ...\n```\n\n## References\n- [Typing spec: Generics](https://typing.python.org/en/latest/spec/generics.html#introduction)", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "invalid-metaclass": { "title": "detects invalid `metaclass=` arguments", "description": "## What it does\nChecks for arguments to `metaclass=` that are invalid.\n\n## Why is this bad?\nPython allows arbitrary expressions to be used as the argument to `metaclass=`.\nThese expressions, however, need to be callable and accept the same arguments\nas `type.__new__`.\n\n## Example\n\n```python\ndef f(): ...\n\n# TypeError: f() takes 0 positional arguments but 3 were given\nclass B(metaclass=f): ...\n```\n\n## References\n- [Python documentation: Metaclasses](https://docs.python.org/3/reference/datamodel.html#metaclasses)",