diff --git a/crates/ruff_memory_usage/src/lib.rs b/crates/ruff_memory_usage/src/lib.rs index e75c75808e..48dbea1540 100644 --- a/crates/ruff_memory_usage/src/lib.rs +++ b/crates/ruff_memory_usage/src/lib.rs @@ -16,13 +16,16 @@ pub fn heap_size(value: &T) -> usize { /// An implementation of [`GetSize::get_heap_size`] for [`OrderSet`]. pub fn order_set_heap_size(set: &OrderSet) -> usize { - (set.capacity() * T::get_stack_size()) + set.iter().map(heap_size).sum::() + let size = set.iter().map(heap_size::).sum::(); + size + (set.capacity() * T::get_stack_size()) } -/// An implementation of [`GetSize::get_heap_size`] for [`OrderMap`]. -pub fn order_map_heap_size(map: &OrderMap) -> usize { - (map.capacity() * (K::get_stack_size() + V::get_stack_size())) - + (map.iter()) - .map(|(k, v)| heap_size(k) + heap_size(v)) - .sum::() +/// An implementation of [`GetSize::get_heap_size`] for [`OrderSet`]. +pub fn order_map_heap_size(set: &OrderMap) -> usize { + let size = set + .iter() + .map(|(key, val)| heap_size::(key) + heap_size::(val)) + .sum::(); + + size + (set.capacity() * <(K, V)>::get_stack_size()) } diff --git a/crates/ty_python_semantic/resources/mdtest/annotations/unsupported_special_types.md b/crates/ty_python_semantic/resources/mdtest/annotations/unsupported_special_types.md index dc22ac2539..8ef5a570a1 100644 --- a/crates/ty_python_semantic/resources/mdtest/annotations/unsupported_special_types.md +++ b/crates/ty_python_semantic/resources/mdtest/annotations/unsupported_special_types.md @@ -10,9 +10,8 @@ import typing MyEnum = enum.Enum("MyEnum", ["foo", "bar", "baz"]) MyIntEnum = enum.IntEnum("MyIntEnum", ["foo", "bar", "baz"]) -MyTypedDict = typing.TypedDict("MyTypedDict", {"foo": int}) MyNamedTuple1 = typing.NamedTuple("MyNamedTuple1", [("foo", int)]) MyNamedTuple2 = collections.namedtuple("MyNamedTuple2", ["foo"]) -def f(a: MyEnum, b: MyTypedDict, c: MyNamedTuple1, d: MyNamedTuple2): ... +def f(a: MyEnum, c: MyNamedTuple1, d: MyNamedTuple2): ... ``` diff --git a/crates/ty_python_semantic/resources/mdtest/typed_dict.md b/crates/ty_python_semantic/resources/mdtest/typed_dict.md index dc3734f368..432cf7053f 100644 --- a/crates/ty_python_semantic/resources/mdtest/typed_dict.md +++ b/crates/ty_python_semantic/resources/mdtest/typed_dict.md @@ -13,6 +13,8 @@ from typing import TypedDict class Person(TypedDict): name: str age: int | None + +reveal_type(Person) # revealed: ``` New inhabitants can be created from dict literals. When accessing keys, the correct types should be @@ -21,6 +23,7 @@ inferred based on the `TypedDict` definition: ```py alice: Person = {"name": "Alice", "age": 30} +reveal_type(alice) # revealed: Person reveal_type(alice["name"]) # revealed: str reveal_type(alice["age"]) # revealed: int | None @@ -85,6 +88,91 @@ alice["extra"] = True bob["extra"] = True ``` +## Functional + +You can also define a `TypedDict` using the functional syntax: + +```py +from typing import TypedDict +from typing_extensions import Required, NotRequired + +Person = TypedDict("Person", {"name": Required[str], "age": int | None}) + +reveal_type(Person) # revealed: typing.TypedDict +``` + +New inhabitants can be created from dict literals. When accessing keys, the correct types should be +inferred based on the `TypedDict` definition: + +```py +alice: Person = {"name": "Alice", "age": 30} + +reveal_type(alice) # revealed: Person +reveal_type(alice["name"]) # revealed: str +reveal_type(alice["age"]) # revealed: int | None + +# error: [invalid-key] "Invalid key access on TypedDict `Person`: Unknown key "non_existing"" +reveal_type(alice["non_existing"]) # revealed: Unknown +``` + +Inhabitants can also be created through a constructor call: + +```py +bob = Person(name="Bob", age=25) + +reveal_type(bob["name"]) # revealed: str +reveal_type(bob["age"]) # revealed: int | None + +# error: [invalid-key] "Invalid key access on TypedDict `Person`: Unknown key "non_existing"" +reveal_type(bob["non_existing"]) # revealed: Unknown +``` + +Methods that are available on `dict`s are also available on `TypedDict`s: + +```py +bob.update(age=26) +``` + +The construction of a `TypedDict` is checked for type correctness: + +```py +# error: [invalid-argument-type] "Invalid argument to key "name" with declared type `str` on TypedDict `Person`" +eve1a: Person = {"name": b"Eve", "age": None} +# error: [invalid-argument-type] "Invalid argument to key "name" with declared type `str` on TypedDict `Person`" +eve1b = Person(name=b"Eve", age=None) + +# error: [missing-typed-dict-key] "Missing required key 'name' in TypedDict `Person` constructor" +eve2a: Person = {"age": 22} +# error: [missing-typed-dict-key] "Missing required key 'name' in TypedDict `Person` constructor" +eve2b = Person(age=22) + +# error: [invalid-key] "Invalid key access on TypedDict `Person`: Unknown key "extra"" +eve3a: Person = {"name": "Eve", "age": 25, "extra": True} +# error: [invalid-key] "Invalid key access on TypedDict `Person`: Unknown key "extra"" +eve3b = Person(name="Eve", age=25, extra=True) +``` + +Assignments to keys are also validated: + +```py +# error: [invalid-assignment] "Invalid assignment to key "name" with declared type `str` on TypedDict `Person`: value of type `None`" +alice["name"] = None + +# error: [invalid-assignment] "Invalid assignment to key "name" with declared type `str` on TypedDict `Person`: value of type `None`" +bob["name"] = None +``` + +Assignments to non-existing keys are disallowed: + +```py +# error: [invalid-key] "Invalid key access on TypedDict `Person`: Unknown key "extra"" +alice["extra"] = True + +# error: [invalid-key] "Invalid key access on TypedDict `Person`: Unknown key "extra"" +bob["extra"] = True +``` + + ## Nested `TypedDict` Nested `TypedDict` fields are also supported. @@ -277,6 +365,29 @@ Extra fields are still not allowed, even with `total=False`: invalid_extra = OptionalPerson(name="George", extra=True) ``` +`total` can also be set with the functional syntax: + +```py +from typing import TypedDict + +OptionalPerson2 = TypedDict("OptionalPerson2", {"name": str, "age": int | None}, total=False) + +charlie = OptionalPerson2() +david = OptionalPerson2(name="David") +emily = OptionalPerson2(age=30) +frank = OptionalPerson2(name="Frank", age=25) + +# TODO: we could emit an error here, because these fields are not guaranteed to exist +reveal_type(charlie["name"]) # revealed: str +reveal_type(david["age"]) # revealed: int | None + +# error: [invalid-argument-type] "Invalid argument to key "name" with declared type `str` on TypedDict `OptionalPerson2`" +invalid = OptionalPerson2(name=123) + +# error: [invalid-key] "Invalid key access on TypedDict `OptionalPerson2`: Unknown key "extra"" +invalid_extra = OptionalPerson2(name="George", extra=True) +``` + ## `Required` and `NotRequired` You can have fine-grained control over keys using `Required` and `NotRequired` qualifiers. These @@ -331,6 +442,42 @@ Type validation still applies to all fields when provided: invalid_type = Message(id="not-an-int", content="Hello") ``` +`Required`/`NotRequired` can also be set with the functional syntax: + +```py +from typing_extensions import TypedDict, Required, NotRequired + +# total=False by default, but id is explicitly Required +Message2 = TypedDict("Message2", {"id": Required[int], "content": str, "timestamp": NotRequired[str]}, total=False) + +# total=True by default, but content is explicitly NotRequired +User2 = TypedDict("User2", {"name": str, "email": Required[str], "bio": NotRequired[str]}) + +# Valid Message2 constructions +msg1 = Message2(id=1) # id required, content optional +msg2 = Message2(id=2, content="Hello") # both provided +msg3 = Message2(id=3, timestamp="2024-01-01") # id required, timestamp optional + +# Valid User2 constructions +user1 = User2(name="Alice", email="alice@example.com") # required fields +user2 = User2(name="Bob", email="bob@example.com", bio="Developer") # with optional bio + +reveal_type(msg1["id"]) # revealed: int +reveal_type(msg1["content"]) # revealed: str +reveal_type(user1["name"]) # revealed: str +reveal_type(user1["bio"]) # revealed: str + +# error: [missing-typed-dict-key] "Missing required key 'id' in TypedDict `Message2` constructor" +invalid_msg = Message2(content="Hello") # Missing required id + +# error: [missing-typed-dict-key] "Missing required key 'name' in TypedDict `User2` constructor" +# error: [missing-typed-dict-key] "Missing required key 'email' in TypedDict `User2` constructor" +invalid_user = User2(bio="No name provided") # Missing required name and email + +# error: [invalid-argument-type] "Invalid argument to key "id" with declared type `int` on TypedDict `Message2`" +invalid_type = Message2(id="not-an-int", content="Hello") +``` + ## Structural assignability Assignability between `TypedDict` types is structural, that is, it is based on the presence of keys @@ -388,6 +535,8 @@ reveal_type(alice["name"]) # revealed: str ### Reading +`class.py`: + ```py from typing import TypedDict, Final, Literal, Any @@ -419,8 +568,41 @@ def _(person: Person, literal_key: Literal["age"], union_of_keys: Literal["age", reveal_type(person[unknown_key]) # revealed: Unknown ``` +`functional.py`: + +```py +from typing import TypedDict, Final, Literal, Any + +Person = TypedDict("Person", {"name": str, "age": int | None}) + +NAME_FINAL: Final = "name" +AGE_FINAL: Final[Literal["age"]] = "age" + +def _(person: Person, literal_key: Literal["age"], union_of_keys: Literal["age", "name"], str_key: str, unknown_key: Any) -> None: + reveal_type(person["name"]) # revealed: str + reveal_type(person["age"]) # revealed: int | None + + reveal_type(person[NAME_FINAL]) # revealed: str + reveal_type(person[AGE_FINAL]) # revealed: int | None + + reveal_type(person[literal_key]) # revealed: int | None + + reveal_type(person[union_of_keys]) # revealed: int | None | str + + # error: [invalid-key] "Invalid key access on TypedDict `Person`: Unknown key "non_existing"" + reveal_type(person["non_existing"]) # revealed: Unknown + + # error: [invalid-key] "TypedDict `Person` cannot be indexed with a key of type `str`" + reveal_type(person[str_key]) # revealed: Unknown + + # No error here: + reveal_type(person[unknown_key]) # revealed: Unknown +``` + ### Writing +`class.py`: + ```py from typing_extensions import TypedDict, Final, Literal, LiteralString, Any @@ -470,10 +652,60 @@ def _(person: Person, unknown_key: Any): person[unknown_key] = "Eve" ``` +`functional.py`: + +```py +from typing_extensions import TypedDict, Final, Literal, LiteralString, Any + +Person = TypedDict("Person", {"name": str, "surname": str, "age": int | None}) + +NAME_FINAL: Final = "name" +AGE_FINAL: Final[Literal["age"]] = "age" + +def _(person: Person): + person["name"] = "Alice" + person["age"] = 30 + + # error: [invalid-key] "Invalid key access on TypedDict `Person`: Unknown key "naem" - did you mean "name"?" + person["naem"] = "Alice" + +def _(person: Person): + person[NAME_FINAL] = "Alice" + person[AGE_FINAL] = 30 + +def _(person: Person, literal_key: Literal["age"]): + person[literal_key] = 22 + +def _(person: Person, union_of_keys: Literal["name", "surname"]): + person[union_of_keys] = "unknown" + + # error: [invalid-assignment] "Cannot assign value of type `Literal[1]` to key of type `Literal["name", "surname"]` on TypedDict `Person`" + person[union_of_keys] = 1 + +def _(person: Person, union_of_keys: Literal["name", "age"], unknown_value: Any): + person[union_of_keys] = unknown_value + + # error: [invalid-assignment] "Cannot assign value of type `None` to key of type `Literal["name", "age"]` on TypedDict `Person`" + person[union_of_keys] = None + +def _(person: Person, str_key: str, literalstr_key: LiteralString): + # error: [invalid-key] "Cannot access `Person` with a key of type `str`. Only string literals are allowed as keys on TypedDicts." + person[str_key] = None + + # error: [invalid-key] "Cannot access `Person` with a key of type `LiteralString`. Only string literals are allowed as keys on TypedDicts." + person[literalstr_key] = None + +def _(person: Person, unknown_key: Any): + # No error here: + person[unknown_key] = "Eve" +``` + ## `ReadOnly` Assignments to keys that are marked `ReadOnly` will produce an error: +`class.py`: + ```py from typing_extensions import TypedDict, ReadOnly, Required @@ -489,10 +721,28 @@ alice["age"] = 31 # okay alice["id"] = 2 ``` +`functional.py`: + +```py +from typing_extensions import TypedDict, ReadOnly, Required + +Person = TypedDict("Person", {"id": ReadOnly[Required[int]], "name": str, "age": int | None}) + +alice: Person = {"id": 1, "name": "Alice", "age": 30} +alice["age"] = 31 # okay + +# error: [invalid-assignment] "Cannot assign to key "id" on TypedDict `Person`: key is marked read-only" +alice["id"] = 2 +``` + This also works if all fields on a `TypedDict` are `ReadOnly`, in which case we synthesize a `__setitem__` method with a `key` type of `Never`: +`never_class.py`: + ```py +from typing_extensions import TypedDict, ReadOnly, Required + class Config(TypedDict): host: ReadOnly[str] port: ReadOnly[int] @@ -505,8 +755,25 @@ config["host"] = "127.0.0.1" config["port"] = 80 ``` +`never_functional.py`: + +```py +from typing_extensions import TypedDict, ReadOnly, Required + +Config = TypedDict("Config", {"host": ReadOnly[str], "port": ReadOnly[int]}) + +config: Config = {"host": "localhost", "port": 8080} + +# error: [invalid-assignment] "Cannot assign to key "host" on TypedDict `Config`: key is marked read-only" +config["host"] = "127.0.0.1" +# error: [invalid-assignment] "Cannot assign to key "port" on TypedDict `Config`: key is marked read-only" +config["port"] = 80 +``` + ## Methods on `TypedDict` +`class.py` + ```py from typing import TypedDict from typing_extensions import NotRequired @@ -556,6 +823,54 @@ def _(p: Person) -> None: reveal_type(p.setdefault("extraz", "value")) # revealed: Unknown ``` +`functional.py`: + +```py +from typing import TypedDict +from typing_extensions import NotRequired + +Person = TypedDict("Person", {"name": str, "age": int | None, "extra": NotRequired[str]}) + +def _(p: Person) -> None: + reveal_type(p.keys()) # revealed: dict_keys[str, object] + reveal_type(p.values()) # revealed: dict_values[str, object] + + # `get()` returns the field type for required keys (no None union) + reveal_type(p.get("name")) # revealed: str + reveal_type(p.get("age")) # revealed: int | None + + # It doesn't matter if a default is specified: + reveal_type(p.get("name", "default")) # revealed: str + reveal_type(p.get("age", 999)) # revealed: int | None + + # `get()` can return `None` for non-required keys + reveal_type(p.get("extra")) # revealed: str | None + reveal_type(p.get("extra", "default")) # revealed: str + + # The type of the default parameter can be anything: + reveal_type(p.get("extra", 0)) # revealed: str | Literal[0] + + # We allow access to unknown keys (they could be set for a subtype of Person) + reveal_type(p.get("unknown")) # revealed: Unknown | None + reveal_type(p.get("unknown", "default")) # revealed: Unknown | Literal["default"] + + # `pop()` only works on non-required fields + reveal_type(p.pop("extra")) # revealed: str + reveal_type(p.pop("extra", "fallback")) # revealed: str + # error: [invalid-argument-type] "Cannot pop required field 'name' from TypedDict `Person`" + reveal_type(p.pop("name")) # revealed: Unknown + + # Similar to above, the default parameter can be of any type: + reveal_type(p.pop("extra", 0)) # revealed: str | Literal[0] + + # `setdefault()` always returns the field type + reveal_type(p.setdefault("name", "Alice")) # revealed: str + reveal_type(p.setdefault("extra", "default")) # revealed: str + + # error: [invalid-key] "Invalid key access on TypedDict `Person`: Unknown key "extraz" - did you mean "extra"?" + reveal_type(p.setdefault("extraz", "value")) # revealed: Unknown +``` + ## Unlike normal classes `TypedDict` types do not act like normal classes. For example, calling `type(..)` on an inhabitant @@ -638,6 +953,26 @@ def accepts_typed_dict_class(t_person: type[Person]) -> None: accepts_typed_dict_class(Person) ``` +Similarly for `TypedDict`s created with the functional syntax: + +```py +Person2 = TypedDict("Person2", {"name": str, "age": int | None}) + +reveal_type(Person2.__total__) # revealed: bool +reveal_type(Person2.__required_keys__) # revealed: frozenset[str] +reveal_type(Person2.__optional_keys__) # revealed: frozenset[str] + +def _(person: Person2) -> None: + person.__total__ # error: [unresolved-attribute] + person.__required_keys__ # error: [unresolved-attribute] + person.__optional_keys__ # error: [unresolved-attribute] + +def _(person: Person2) -> None: + type(person).__total__ # error: [unresolved-attribute] + type(person).__required_keys__ # error: [unresolved-attribute] + type(person).__optional_keys__ # error: [unresolved-attribute] +``` + ## Subclassing `TypedDict` types can be subclassed. The subclass can add new keys: @@ -801,27 +1136,6 @@ nested: Node = {"name": "n1", "parent": {"name": "n2", "parent": {"name": "n3", nested_invalid: Node = {"name": "n1", "parent": {"name": "n2", "parent": {"name": 3, "parent": None}}} ``` -## Function/assignment syntax - -This is not yet supported. Make sure that we do not emit false positives for this syntax: - -```py -from typing_extensions import TypedDict, Required - -# Alternative syntax -Message = TypedDict("Message", {"id": Required[int], "content": str}, total=False) - -msg = Message(id=1, content="Hello") - -# No errors for yet-unsupported features (`closed`): -OtherMessage = TypedDict("OtherMessage", {"id": int, "content": str}, closed=True) - -reveal_type(Message.__required_keys__) # revealed: @Todo(Support for `TypedDict`) - -# TODO: this should be an error -msg.content -``` - ## Error cases ### `typing.TypedDict` is not allowed in type expressions diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 9d0e04bbba..41f1d172dd 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -1009,6 +1009,17 @@ impl<'db> Type<'db> { } } + /// Turn a typed dict class literal or synthesized type dict type into a `TypedDictType`. + pub(crate) fn to_typed_dict_type(self, db: &'db dyn Db) -> Option> { + match self { + Type::ClassLiteral(class_literal) if class_literal.is_typed_dict(db) => Some( + TypedDictType::from_class(ClassType::NonGeneric(class_literal)), + ), + Type::KnownInstance(KnownInstanceType::TypedDictType(typed_dict)) => Some(typed_dict), + _ => None, + } + } + /// Turn a class literal (`Type::ClassLiteral` or `Type::GenericAlias`) into a `ClassType`. /// Since a `ClassType` must be specialized, apply the default specialization to any /// unspecialized generic class literal. @@ -1127,7 +1138,7 @@ impl<'db> Type<'db> { } pub(crate) fn typed_dict(defining_class: impl Into>) -> Self { - Self::TypedDict(TypedDictType::new(defining_class.into())) + Self::TypedDict(TypedDictType::from_class(defining_class.into())) } #[must_use] @@ -1253,10 +1264,9 @@ impl<'db> Type<'db> { // Always normalize single-member enums to their class instance (`Literal[Single.VALUE]` => `Single`) enum_literal.enum_class_instance(db) } - Type::TypedDict(_) => { - // TODO: Normalize TypedDicts - self - } + Type::TypedDict(typed_dict) => visitor.visit(self, || { + Type::TypedDict(typed_dict.normalized_impl(db, visitor)) + }), Type::TypeAlias(alias) => alias.value_type(db).normalized_impl(db, visitor), Type::LiteralString | Type::AlwaysFalsy @@ -2974,6 +2984,14 @@ impl<'db> Type<'db> { policy: MemberLookupPolicy, ) -> PlaceAndQualifiers<'db> { tracing::trace!("class_member: {}.{}", self.display(db), name); + + if let Type::TypedDict(TypedDictType::Synthesized(typed_dict)) = self + && let Some(member) = + TypedDictType::synthesized_member(db, self, typed_dict.items(db), &name) + { + return Place::bound(member).into(); + } + match self { Type::Union(union) => union.map_with_boundness_and_qualifiers(db, |elem| { elem.class_member_with_policy(db, name.clone(), policy) @@ -3728,7 +3746,7 @@ impl<'db> Type<'db> { } Type::ClassLiteral(..) | Type::GenericAlias(..) | Type::SubclassOf(..) => { - let class_attr_plain = self.find_name_in_mro_with_policy(db, name_str,policy).expect( + let class_attr_plain = self.find_name_in_mro_with_policy(db, name_str, policy).expect( "Calling `find_name_in_mro` on class literals and subclass-of types should always return `Some`", ); @@ -4746,7 +4764,7 @@ impl<'db> Type<'db> { Parameter::positional_only(Some(Name::new_static("typename"))) .with_annotated_type(KnownClass::Str.to_instance(db)), Parameter::positional_only(Some(Name::new_static("fields"))) - .with_annotated_type(KnownClass::Dict.to_instance(db)) + .with_annotated_type(Type::SpecialForm(SpecialFormType::TypedDict)) .with_default_type(Type::any()), Parameter::keyword_only(Name::new_static("total")) .with_annotated_type(KnownClass::Bool.to_instance(db)) @@ -4765,6 +4783,8 @@ impl<'db> Type<'db> { Binding::single(self, Signature::todo("functional `NamedTuple` syntax")).into() } + Type::SpecialForm(_) => CallableBinding::not_callable(self).into(), + Type::GenericAlias(_) => { // TODO annotated return type on `__new__` or metaclass `__call__` // TODO check call vs signatures of `__new__` and/or `__init__` @@ -4833,11 +4853,18 @@ impl<'db> Type<'db> { // TODO: this is actually callable Type::DataclassDecorator(_) => CallableBinding::not_callable(self).into(), - // TODO: some `SpecialForm`s are callable (e.g. TypedDicts) - Type::SpecialForm(_) => CallableBinding::not_callable(self).into(), - Type::EnumLiteral(enum_literal) => enum_literal.enum_class_instance(db).bindings(db), + Type::KnownInstance(KnownInstanceType::TypedDictType(typed_dict)) => Binding::single( + self, + Signature::new( + Parameters::new([Parameter::keyword_variadic(Name::new_static("kwargs")) + .with_annotated_type(Type::any())]), + Some(Type::TypedDict(typed_dict)), + ), + ) + .into(), + Type::KnownInstance(known_instance) => { known_instance.instance_fallback(db).bindings(db) } @@ -5641,6 +5668,7 @@ impl<'db> Type<'db> { .map(Type::NonInferableTypeVar) .unwrap_or(*self)) } + KnownInstanceType::TypedDictType(typed_dict) => Ok(Type::TypedDict(*typed_dict)), KnownInstanceType::Deprecated(_) => Err(InvalidTypeExpressionError { invalid_expressions: smallvec::smallvec![InvalidTypeExpression::Deprecated], fallback_type: Type::unknown(), @@ -5927,9 +5955,7 @@ impl<'db> Type<'db> { Type::AlwaysTruthy | Type::AlwaysFalsy => KnownClass::Type.to_instance(db), Type::BoundSuper(_) => KnownClass::Super.to_class_literal(db), Type::ProtocolInstance(protocol) => protocol.to_meta_type(db), - // `TypedDict` instances are instances of `dict` at runtime, but its important that we - // understand a more specific meta type in order to correctly handle `__getitem__`. - Type::TypedDict(typed_dict) => SubclassOfType::from(db, typed_dict.defining_class()), + Type::TypedDict(typed_dict) => typed_dict.to_meta_type(db), Type::TypeAlias(alias) => alias.value_type(db).to_meta_type(db), } } @@ -6486,8 +6512,9 @@ impl<'db> Type<'db> { Protocol::Synthesized(_) => None, }, - Type::TypedDict(typed_dict) => { - Some(TypeDefinition::Class(typed_dict.defining_class().definition(db))) + Type::TypedDict(typed_dict) => match typed_dict { + TypedDictType::FromClass(class) => Some(TypeDefinition::Class(class.definition(db))), + TypedDictType::Synthesized(_) => None, } Self::Union(_) | Self::Intersection(_) => None, @@ -6836,6 +6863,9 @@ pub enum KnownInstanceType<'db> { /// A single instance of `typing.TypeAliasType` (PEP 695 type alias) TypeAliasType(TypeAliasType<'db>), + /// A single instance of `typing.TypedDict`. + TypedDictType(TypedDictType<'db>), + /// A single instance of `warnings.deprecated` or `typing_extensions.deprecated` Deprecated(DeprecatedInstance<'db>), @@ -6863,6 +6893,9 @@ fn walk_known_instance_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>( KnownInstanceType::TypeAliasType(type_alias) => { visitor.visit_type_alias_type(db, type_alias); } + KnownInstanceType::TypedDictType(typed_dict) => { + visitor.visit_typed_dict_type(db, typed_dict); + } KnownInstanceType::Deprecated(_) | KnownInstanceType::ConstraintSet(_) => { // Nothing to visit } @@ -6885,6 +6918,9 @@ impl<'db> KnownInstanceType<'db> { Self::TypeAliasType(type_alias) => { Self::TypeAliasType(type_alias.normalized_impl(db, visitor)) } + Self::TypedDictType(typed_dict) => { + Self::TypedDictType(typed_dict.normalized_impl(db, visitor)) + } Self::Deprecated(deprecated) => { // Nothing to normalize Self::Deprecated(deprecated) @@ -6905,6 +6941,7 @@ impl<'db> KnownInstanceType<'db> { KnownClass::GenericAlias } Self::TypeAliasType(_) => KnownClass::TypeAliasType, + Self::TypedDictType(_) => KnownClass::TypedDictFallback, Self::Deprecated(_) => KnownClass::Deprecated, Self::Field(_) => KnownClass::Field, Self::ConstraintSet(_) => KnownClass::ConstraintSet, @@ -6961,6 +6998,7 @@ impl<'db> KnownInstanceType<'db> { f.write_str("typing.TypeAliasType") } } + KnownInstanceType::TypedDictType(_) => f.write_str("typing.TypedDict"), // This is a legacy `TypeVar` _outside_ of any generic class or function, so we render // it as an instance of `typing.TypeVar`. Inside of a generic class or function, we'll // have a `Type::TypeVar(_)`, which is rendered as the typevar's name. @@ -9119,7 +9157,7 @@ impl TypeRelation { } } -#[derive(Debug, Copy, Clone, PartialEq, Eq, get_size2::GetSize)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, get_size2::GetSize)] pub enum Truthiness { /// For an object `x`, `bool(x)` will always return `True` AlwaysTrue, @@ -9166,6 +9204,14 @@ impl Truthiness { } } + pub(crate) fn unwrap_or(self, value: bool) -> bool { + match self { + Truthiness::AlwaysTrue => true, + Truthiness::AlwaysFalse => false, + Truthiness::Ambiguous => value, + } + } + fn into_type(self, db: &dyn Db) -> Type<'_> { match self { Self::AlwaysTrue => Type::BooleanLiteral(true), diff --git a/crates/ty_python_semantic/src/types/call/bind.rs b/crates/ty_python_semantic/src/types/call/bind.rs index 60a0394ec8..6b9ba4b5dc 100644 --- a/crates/ty_python_semantic/src/types/call/bind.rs +++ b/crates/ty_python_semantic/src/types/call/bind.rs @@ -12,11 +12,11 @@ use ruff_python_ast::name::Name; use smallvec::{SmallVec, smallvec, smallvec_inline}; use super::{Argument, CallArguments, CallError, CallErrorKind, InferContext, Signature, Type}; -use crate::Program; use crate::db::Db; use crate::dunder_all::dunder_all_names; use crate::place::{Boundness, Place}; use crate::types::call::arguments::{Expansion, is_expandable_type}; +use crate::types::class::{Field, FieldKind}; use crate::types::diagnostic::{ CALL_NON_CALLABLE, CONFLICTING_ARGUMENT_FORMS, INVALID_ARGUMENT_TYPE, MISSING_ARGUMENT, NO_MATCHING_OVERLOAD, PARAMETER_ALREADY_ASSIGNED, POSITIONAL_ONLY_PARAMETER_AS_KWARG, @@ -29,12 +29,14 @@ use crate::types::function::{ use crate::types::generics::{Specialization, SpecializationBuilder, SpecializationError}; use crate::types::signatures::{Parameter, ParameterForm, ParameterKind, Parameters}; use crate::types::tuple::{TupleLength, TupleType}; +use crate::types::typed_dict::SynthesizedTypedDictType; use crate::types::{ BoundMethodType, ClassLiteral, DataclassParams, FieldInstance, KnownBoundMethodType, KnownClass, KnownInstanceType, MemberLookupPolicy, PropertyInstanceType, SpecialFormType, - TrackedConstraintSet, TypeAliasType, TypeContext, UnionBuilder, UnionType, - WrapperDescriptorKind, enums, ide_support, infer_isolated_expression, todo_type, + TrackedConstraintSet, TypeAliasType, TypeContext, TypedDictParams, TypedDictType, UnionBuilder, + UnionType, WrapperDescriptorKind, enums, ide_support, infer_isolated_expression, }; +use crate::{FxOrderMap, Program}; use ruff_db::diagnostic::{Annotation, Diagnostic, SubDiagnostic, SubDiagnosticSeverity}; use ruff_python_ast::{self as ast, ArgOrKeyword, PythonVersion}; @@ -1096,7 +1098,57 @@ impl<'db> Bindings<'db> { }, Type::SpecialForm(SpecialFormType::TypedDict) => { - overload.set_return_type(todo_type!("Support for `TypedDict`")); + let [Some(name), Some(Type::TypedDict(typed_dict)), total, ..] = + overload.parameter_types() + else { + continue; + }; + + let Some(name) = name.into_string_literal() else { + continue; + }; + + let mut params = TypedDictParams::empty(); + + let is_total = to_bool(total, true); + params.set(TypedDictParams::TOTAL, is_total); + + let items = typed_dict.items(db); + let items = items + .iter() + .map(|(name, field)| { + let FieldKind::TypedDict { + is_required, + is_read_only, + } = field.kind + else { + unreachable!() + }; + + let field = Field { + kind: FieldKind::TypedDict { + is_read_only, + // If there is no explicit `Required`/`NotRequired` qualifier, use + // the `total` parameter. + is_required: is_required.unwrap_or(is_total).into(), + }, + ..field.clone() + }; + + (name.clone(), field) + }) + .collect::>(); + + overload.set_return_type(Type::KnownInstance( + KnownInstanceType::TypedDictType(TypedDictType::Synthesized( + SynthesizedTypedDictType::new( + db, + Some(Name::new(name.value(db))), + params, + items, + ), + )), + )); } // Not a special case @@ -2346,7 +2398,7 @@ impl<'a, 'db> ArgumentMatcher<'a, 'db> { ) { if let Some(Type::TypedDict(typed_dict)) = argument_type { // Special case TypedDict because we know which keys are present. - for (name, field) in typed_dict.items(db) { + for (name, field) in typed_dict.items(db).as_ref() { let _ = self.match_keyword( argument_index, Argument::Keywords, diff --git a/crates/ty_python_semantic/src/types/class.rs b/crates/ty_python_semantic/src/types/class.rs index ae44c0cebd..a59c2b3cc3 100644 --- a/crates/ty_python_semantic/src/types/class.rs +++ b/crates/ty_python_semantic/src/types/class.rs @@ -28,9 +28,9 @@ use crate::types::{ ApplyTypeMappingVisitor, Binding, BoundSuperError, BoundSuperType, CallableType, DataclassParams, DeprecatedInstance, FindLegacyTypeVarsVisitor, HasRelationToVisitor, IsEquivalentVisitor, KnownInstanceType, ManualPEP695TypeAliasType, MaterializationKind, - NormalizedVisitor, PropertyInstanceType, StringLiteralType, TypeAliasType, TypeContext, - TypeMapping, TypeRelation, TypeVarBoundOrConstraints, TypeVarInstance, TypeVarKind, - TypedDictParams, UnionBuilder, VarianceInferable, declaration_type, determine_upper_bound, + NormalizedVisitor, PropertyInstanceType, TypeAliasType, TypeContext, TypeMapping, TypeRelation, + TypeVarBoundOrConstraints, TypeVarInstance, TypeVarKind, TypedDictParams, TypedDictType, + UnionBuilder, VarianceInferable, declaration_type, determine_upper_bound, infer_definition_types, }; use crate::{ @@ -1267,7 +1267,7 @@ impl MethodDecorator { } /// Kind-specific metadata for different types of fields -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, get_size2::GetSize)] pub(crate) enum FieldKind<'db> { /// `NamedTuple` field metadata NamedTuple { default_ty: Option> }, @@ -1286,15 +1286,15 @@ pub(crate) enum FieldKind<'db> { /// `TypedDict` field metadata TypedDict { /// Whether this field is required - is_required: bool, + is_required: Truthiness, /// Whether this field is marked read-only is_read_only: bool, }, } /// Metadata regarding a dataclass field/attribute or a `TypedDict` "item" / key-value pair. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct Field<'db> { +#[derive(Debug, Clone, PartialEq, Eq, Hash, get_size2::GetSize)] +pub struct Field<'db> { /// The declared type of the field pub(crate) declared_ty: Type<'db>, /// Kind-specific metadata for this field @@ -1304,7 +1304,7 @@ pub(crate) struct Field<'db> { pub(crate) single_declaration: Option>, } -impl Field<'_> { +impl<'db> Field<'db> { pub(crate) const fn is_required(&self) -> bool { match &self.kind { FieldKind::NamedTuple { default_ty } => default_ty.is_none(), @@ -1313,7 +1313,7 @@ impl Field<'_> { FieldKind::Dataclass { init, default_ty, .. } => default_ty.is_none() && *init, - FieldKind::TypedDict { is_required, .. } => *is_required, + FieldKind::TypedDict { is_required, .. } => is_required.is_always_true(), } } @@ -1323,6 +1323,21 @@ impl Field<'_> { _ => false, } } + + pub(super) fn apply_type_mapping_impl<'a>( + self, + db: &'db dyn Db, + type_mapping: &TypeMapping<'a, 'db>, + visitor: &ApplyTypeMappingVisitor<'db>, + ) -> Self { + Field { + kind: self.kind, + single_declaration: self.single_declaration, + declared_ty: self + .declared_ty + .apply_type_mapping_impl(db, type_mapping, visitor), + } + } } impl<'db> Field<'db> { @@ -2360,273 +2375,12 @@ impl<'db> ClassLiteral<'db> { Type::heterogeneous_tuple(db, slots) }) } - (CodeGeneratorKind::TypedDict, "__setitem__") => { + + (CodeGeneratorKind::TypedDict, _) => { let fields = self.fields(db, specialization, field_policy); - - // Add (key type, value type) overloads for all TypedDict items ("fields") that are not read-only: - - let mut writeable_fields = fields - .iter() - .filter(|(_, field)| !field.is_read_only()) - .peekable(); - - if writeable_fields.peek().is_none() { - // If there are no writeable fields, synthesize a `__setitem__` that takes - // a `key` of type `Never` to signal that no keys are accepted. This leads - // to slightly more user-friendly error messages compared to returning an - // empty overload set. - return Some(Type::Callable(CallableType::new( - db, - CallableSignature::single(Signature::new( - Parameters::new([ - Parameter::positional_only(Some(Name::new_static("self"))) - .with_annotated_type(instance_ty), - Parameter::positional_only(Some(Name::new_static("key"))) - .with_annotated_type(Type::Never), - Parameter::positional_only(Some(Name::new_static("value"))) - .with_annotated_type(Type::any()), - ]), - Some(Type::none(db)), - )), - true, - ))); - } - - let overloads = writeable_fields.map(|(name, field)| { - let key_type = Type::StringLiteral(StringLiteralType::new(db, name.as_str())); - - Signature::new( - Parameters::new([ - Parameter::positional_only(Some(Name::new_static("self"))) - .with_annotated_type(instance_ty), - Parameter::positional_only(Some(Name::new_static("key"))) - .with_annotated_type(key_type), - Parameter::positional_only(Some(Name::new_static("value"))) - .with_annotated_type(field.declared_ty), - ]), - Some(Type::none(db)), - ) - }); - - Some(Type::Callable(CallableType::new( - db, - CallableSignature::from_overloads(overloads), - true, - ))) + TypedDictType::synthesized_member(db, instance_ty, &fields, name) } - (CodeGeneratorKind::TypedDict, "__getitem__") => { - let fields = self.fields(db, specialization, field_policy); - // Add (key -> value type) overloads for all TypedDict items ("fields"): - let overloads = fields.iter().map(|(name, field)| { - let key_type = Type::StringLiteral(StringLiteralType::new(db, name.as_str())); - - Signature::new( - Parameters::new([ - Parameter::positional_only(Some(Name::new_static("self"))) - .with_annotated_type(instance_ty), - Parameter::positional_only(Some(Name::new_static("key"))) - .with_annotated_type(key_type), - ]), - Some(field.declared_ty), - ) - }); - - Some(Type::Callable(CallableType::new( - db, - CallableSignature::from_overloads(overloads), - true, - ))) - } - (CodeGeneratorKind::TypedDict, "get") => { - let overloads = self - .fields(db, specialization, field_policy) - .into_iter() - .flat_map(|(name, field)| { - let key_type = - Type::StringLiteral(StringLiteralType::new(db, name.as_str())); - - // For a required key, `.get()` always returns the value type. For a non-required key, - // `.get()` returns the union of the value type and the type of the default argument - // (which defaults to `None`). - - // TODO: For now, we use two overloads here. They can be merged into a single function - // once the generics solver takes default arguments into account. - - let get_sig = Signature::new( - Parameters::new([ - Parameter::positional_only(Some(Name::new_static("self"))) - .with_annotated_type(instance_ty), - Parameter::positional_only(Some(Name::new_static("key"))) - .with_annotated_type(key_type), - ]), - Some(if field.is_required() { - field.declared_ty - } else { - UnionType::from_elements(db, [field.declared_ty, Type::none(db)]) - }), - ); - - let t_default = - BoundTypeVarInstance::synthetic(db, "T", TypeVarVariance::Covariant); - - let get_with_default_sig = Signature::new_generic( - Some(GenericContext::from_typevar_instances(db, [t_default])), - Parameters::new([ - Parameter::positional_only(Some(Name::new_static("self"))) - .with_annotated_type(instance_ty), - Parameter::positional_only(Some(Name::new_static("key"))) - .with_annotated_type(key_type), - Parameter::positional_only(Some(Name::new_static("default"))) - .with_annotated_type(Type::TypeVar(t_default)), - ]), - Some(if field.is_required() { - field.declared_ty - } else { - UnionType::from_elements( - db, - [field.declared_ty, Type::TypeVar(t_default)], - ) - }), - ); - - [get_sig, get_with_default_sig] - }) - // Fallback overloads for unknown keys - .chain(std::iter::once({ - Signature::new( - Parameters::new([ - Parameter::positional_only(Some(Name::new_static("self"))) - .with_annotated_type(instance_ty), - Parameter::positional_only(Some(Name::new_static("key"))) - .with_annotated_type(KnownClass::Str.to_instance(db)), - ]), - Some(UnionType::from_elements( - db, - [Type::unknown(), Type::none(db)], - )), - ) - })) - .chain(std::iter::once({ - let t_default = - BoundTypeVarInstance::synthetic(db, "T", TypeVarVariance::Covariant); - - Signature::new_generic( - Some(GenericContext::from_typevar_instances(db, [t_default])), - Parameters::new([ - Parameter::positional_only(Some(Name::new_static("self"))) - .with_annotated_type(instance_ty), - Parameter::positional_only(Some(Name::new_static("key"))) - .with_annotated_type(KnownClass::Str.to_instance(db)), - Parameter::positional_only(Some(Name::new_static("default"))) - .with_annotated_type(Type::TypeVar(t_default)), - ]), - Some(UnionType::from_elements( - db, - [Type::unknown(), Type::TypeVar(t_default)], - )), - ) - })); - - Some(Type::Callable(CallableType::new( - db, - CallableSignature::from_overloads(overloads), - true, - ))) - } - (CodeGeneratorKind::TypedDict, "pop") => { - let fields = self.fields(db, specialization, field_policy); - let overloads = fields - .iter() - .filter(|(_, field)| { - // Only synthesize `pop` for fields that are not required. - !field.is_required() - }) - .flat_map(|(name, field)| { - let key_type = - Type::StringLiteral(StringLiteralType::new(db, name.as_str())); - - // TODO: Similar to above: consider merging these two overloads into one - - // `.pop()` without default - let pop_sig = Signature::new( - Parameters::new([ - Parameter::positional_only(Some(Name::new_static("self"))) - .with_annotated_type(instance_ty), - Parameter::positional_only(Some(Name::new_static("key"))) - .with_annotated_type(key_type), - ]), - Some(field.declared_ty), - ); - - // `.pop()` with a default value - let t_default = - BoundTypeVarInstance::synthetic(db, "T", TypeVarVariance::Covariant); - - let pop_with_default_sig = Signature::new_generic( - Some(GenericContext::from_typevar_instances(db, [t_default])), - Parameters::new([ - Parameter::positional_only(Some(Name::new_static("self"))) - .with_annotated_type(instance_ty), - Parameter::positional_only(Some(Name::new_static("key"))) - .with_annotated_type(key_type), - Parameter::positional_only(Some(Name::new_static("default"))) - .with_annotated_type(Type::TypeVar(t_default)), - ]), - Some(UnionType::from_elements( - db, - [field.declared_ty, Type::TypeVar(t_default)], - )), - ); - - [pop_sig, pop_with_default_sig] - }); - - Some(Type::Callable(CallableType::new( - db, - CallableSignature::from_overloads(overloads), - true, - ))) - } - (CodeGeneratorKind::TypedDict, "setdefault") => { - let fields = self.fields(db, specialization, field_policy); - let overloads = fields.iter().map(|(name, field)| { - let key_type = Type::StringLiteral(StringLiteralType::new(db, name.as_str())); - - // `setdefault` always returns the field type - Signature::new( - Parameters::new([ - Parameter::positional_only(Some(Name::new_static("self"))) - .with_annotated_type(instance_ty), - Parameter::positional_only(Some(Name::new_static("key"))) - .with_annotated_type(key_type), - Parameter::positional_only(Some(Name::new_static("default"))) - .with_annotated_type(field.declared_ty), - ]), - Some(field.declared_ty), - ) - }); - - Some(Type::Callable(CallableType::new( - db, - CallableSignature::from_overloads(overloads), - true, - ))) - } - (CodeGeneratorKind::TypedDict, "update") => { - // TODO: synthesize a set of overloads with precise types - let signature = Signature::new( - Parameters::new([ - Parameter::positional_only(Some(Name::new_static("self"))) - .with_annotated_type(instance_ty), - Parameter::variadic(Name::new_static("args")), - Parameter::keyword_variadic(Name::new_static("kwargs")), - ]), - Some(Type::none(db)), - ); - - Some(CallableType::function_like(db, signature)) - } _ => None, } } @@ -2812,7 +2566,7 @@ impl<'db> ClassLiteral<'db> { }; FieldKind::TypedDict { - is_required, + is_required: Truthiness::from(is_required), is_read_only: attr.is_read_only(), } } diff --git a/crates/ty_python_semantic/src/types/class_base.rs b/crates/ty_python_semantic/src/types/class_base.rs index 547ace5923..25ad8fb796 100644 --- a/crates/ty_python_semantic/src/types/class_base.rs +++ b/crates/ty_python_semantic/src/types/class_base.rs @@ -169,6 +169,7 @@ impl<'db> ClassBase<'db> { KnownInstanceType::SubscriptedProtocol(_) => Some(Self::Protocol), KnownInstanceType::TypeAliasType(_) | KnownInstanceType::TypeVar(_) + | KnownInstanceType::TypedDictType(_) | KnownInstanceType::Deprecated(_) | KnownInstanceType::Field(_) | KnownInstanceType::ConstraintSet(_) => None, diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index f20ebefd21..bf05e462bb 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -23,7 +23,7 @@ use crate::types::visitor::TypeVisitor; use crate::types::{ BoundTypeVarInstance, CallableType, IntersectionType, KnownBoundMethodType, KnownClass, MaterializationKind, Protocol, ProtocolInstanceType, StringLiteralType, SubclassOfInner, Type, - UnionType, WrapperDescriptorKind, visitor, + TypedDictType, UnionType, WrapperDescriptorKind, visitor, }; use ruff_db::parsed::parsed_module; @@ -510,12 +510,22 @@ impl Display for DisplayRepresentation<'_> { } f.write_str("]") } - Type::TypedDict(typed_dict) => typed_dict - .defining_class() - .class_literal(self.db) - .0 - .display_with(self.db, self.settings.clone()) - .fmt(f), + + Type::TypedDict(typed_dict) => match typed_dict { + TypedDictType::FromClass(class) => class + .class_literal(self.db) + .0 + .display_with(self.db, self.settings.clone()) + .fmt(f), + TypedDictType::Synthesized(synthesized) => { + let name = synthesized + .name(self.db) + .expect("cannot have incomplete `TypedDict` in type expression"); + + write!(f, "{name}") + } + }, + Type::TypeAlias(alias) => f.write_str(alias.name(self.db)), } } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 94a823c8c1..17d921ebb9 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -46,7 +46,9 @@ use crate::semantic_index::{ }; use crate::types::call::bind::MatchingOverloadIndex; use crate::types::call::{Binding, Bindings, CallArguments, CallError, CallErrorKind}; -use crate::types::class::{CodeGeneratorKind, FieldKind, MetaclassErrorKind, MethodDecorator}; +use crate::types::class::{ + CodeGeneratorKind, Field, FieldKind, MetaclassErrorKind, MethodDecorator, +}; use crate::types::context::{InNoTypeCheck, InferContext}; use crate::types::cyclic::CycleDetector; use crate::types::diagnostic::{ @@ -101,7 +103,7 @@ use crate::types::{ClassBase, add_inferred_python_version_hint_to_diagnostic}; use crate::unpack::{EvaluationMode, UnpackPosition}; use crate::util::diagnostics::format_enumeration; use crate::util::subscript::{PyIndex, PySlice}; -use crate::{Db, FxOrderSet, Program}; +use crate::{Db, FxOrderMap, FxOrderSet, Program}; mod annotation_expression; mod type_expression; @@ -5472,9 +5474,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } = dict; // Validate `TypedDict` dictionary literal assignments. - if let Some(typed_dict) = tcx.annotation.and_then(Type::into_typed_dict) - && let Some(ty) = self.infer_typed_dict_expression(dict, typed_dict) - { + if let Some(ty) = self.infer_typed_dict_expression(dict, tcx) { return ty; } @@ -5496,39 +5496,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { }) } - fn infer_typed_dict_expression( - &mut self, - dict: &ast::ExprDict, - typed_dict: TypedDictType<'db>, - ) -> Option> { - let ast::ExprDict { - range: _, - node_index: _, - items, - } = dict; - - let typed_dict_items = typed_dict.items(self.db()); - - for item in items { - self.infer_optional_expression(item.key.as_ref(), TypeContext::default()); - - if let Some(ast::Expr::StringLiteral(ref key)) = item.key - && let Some(key) = key.as_single_part_string() - && let Some(field) = typed_dict_items.get(key.as_str()) - { - self.infer_expression(&item.value, TypeContext::new(Some(field.declared_ty))); - } else { - self.infer_expression(&item.value, TypeContext::default()); - } - } - - validate_typed_dict_dict_literal(&self.context, typed_dict, dict, dict.into(), |expr| { - self.expression_type(expr) - }) - .ok() - .map(|_| Type::TypedDict(typed_dict)) - } - // Infer the type of a collection literal expression. fn infer_collection_literal<'expr, const N: usize>( &mut self, @@ -5625,6 +5592,165 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Type::from(class_type).to_instance(self.db()) } + fn infer_typed_dict_expression( + &mut self, + dict: &ast::ExprDict, + tcx: TypeContext<'db>, + ) -> Option> { + let ast::ExprDict { + range: _, + node_index: _, + items, + } = dict; + + // Evaluate the dictionary literal passed to the `TypedDict` constructor. + if let Some(Type::SpecialForm(SpecialFormType::TypedDict)) = tcx.annotation { + let mut typed_dict_items = FxOrderMap::default(); + + for item in items { + let Some(Type::StringLiteral(key)) = + self.infer_optional_expression(item.key.as_ref(), TypeContext::default()) + else { + // Emit a diagnostic here? We seem to support non-string literals. + unimplemented!() + }; + + let field_ty = self.infer_typed_dict_field_type_expression(&item.value); + + let is_required = if field_ty.qualifiers.contains(TypeQualifiers::REQUIRED) { + // Explicit Required[T] annotation - always required + Truthiness::AlwaysTrue + } else if field_ty.qualifiers.contains(TypeQualifiers::NOT_REQUIRED) { + // Explicit NotRequired[T] annotation - never required + Truthiness::AlwaysFalse + } else { + // No explicit qualifier - we don't have access to the `total` qualifier here, + // so we leave this to be filled in by the `TypedDict` constructor. + Truthiness::Ambiguous + }; + + let field = Field { + single_declaration: None, + declared_ty: field_ty.inner_type(), + kind: FieldKind::TypedDict { + is_required, + is_read_only: field_ty.qualifiers.contains(TypeQualifiers::READ_ONLY), + }, + }; + + typed_dict_items.insert(ast::name::Name::new(key.value(self.db())), field); + } + + // Create an incomplete synthesized `TypedDictType`, to be completed by the `TypedDict` + // constructor binding. + return Some(Type::TypedDict(TypedDictType::from_items( + self.db(), + typed_dict_items, + ))); + } + + let typed_dict = tcx.annotation.and_then(Type::into_typed_dict)?; + let typed_dict_items = typed_dict.items(self.db()); + + for item in items { + self.infer_optional_expression(item.key.as_ref(), TypeContext::default()); + + if let Some(ast::Expr::StringLiteral(ref key)) = item.key + && let Some(key) = key.as_single_part_string() + && let Some(field) = typed_dict_items.get(key.as_str()) + { + self.infer_expression(&item.value, TypeContext::new(Some(field.declared_ty))); + } else { + self.infer_expression(&item.value, TypeContext::default()); + } + } + + validate_typed_dict_dict_literal(&self.context, typed_dict, dict, dict.into(), |expr| { + self.expression_type(expr) + }) + .ok() + .map(|_| Type::TypedDict(typed_dict)) + } + + fn infer_typed_dict_field_type_expression( + &mut self, + expr: &ast::Expr, + ) -> TypeAndQualifiers<'db> { + let ty = match expr { + ast::Expr::Subscript(subscript @ ast::ExprSubscript { value, slice, .. }) => { + let value_ty = self.infer_expression(value, TypeContext::default()); + let slice = &**slice; + + // Unlike other type-form expressions, `TypedDict` constructor literals support + // the `Required`, `NotRequired`, and `ReadOnly` qualifiers. + match value_ty { + Type::SpecialForm( + type_qualifier @ (SpecialFormType::Required + | SpecialFormType::NotRequired + | SpecialFormType::ReadOnly), + ) => { + let arguments = if let ast::Expr::Tuple(tuple) = slice { + &*tuple.elts + } else { + std::slice::from_ref(slice) + }; + + let num_arguments = arguments.len(); + let type_and_qualifiers = if num_arguments == 1 { + let mut type_and_qualifiers = + self.infer_typed_dict_field_type_expression(slice); + + match type_qualifier { + SpecialFormType::Required => { + type_and_qualifiers.add_qualifier(TypeQualifiers::REQUIRED); + } + SpecialFormType::NotRequired => { + type_and_qualifiers.add_qualifier(TypeQualifiers::NOT_REQUIRED); + } + SpecialFormType::ReadOnly => { + type_and_qualifiers.add_qualifier(TypeQualifiers::READ_ONLY); + } + _ => unreachable!(), + } + + type_and_qualifiers + } else { + for element in arguments { + self.infer_typed_dict_field_type_expression(element); + } + + if let Some(builder) = + self.context.report_lint(&INVALID_TYPE_FORM, subscript) + { + builder.into_diagnostic(format_args!( + "Type qualifier `{type_qualifier}` expected exactly 1 argument, \ + got {num_arguments}", + )); + } + + Type::unknown().into() + }; + + if slice.is_tuple_expr() { + self.store_expression_type(slice, type_and_qualifiers.inner_type()); + } + + type_and_qualifiers + } + + _ => self + .infer_subscript_type_expression_no_store(subscript, slice, value_ty) + .into(), + } + } + + type_expr => self.infer_type_expression_no_store(type_expr).into(), + }; + + self.store_expression_type(expr, ty.inner_type()); + ty + } + /// Infer the type of the `iter` expression of the first comprehension. fn infer_first_comprehension_iter(&mut self, comprehensions: &[ast::Comprehension]) { let mut comprehensions_iter = comprehensions.iter(); @@ -6013,7 +6139,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ty }); - // TODO: Use the type context for more precise inference. let callable_type = self.infer_maybe_standalone_expression(func, TypeContext::default()); // Special handling for `TypedDict` method calls @@ -6157,19 +6282,14 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.infer_all_argument_types(arguments, &mut call_arguments, &bindings); // Validate `TypedDict` constructor calls after argument type inference - if let Some(class_literal) = callable_type.into_class_literal() { - if class_literal.is_typed_dict(self.db()) { - let typed_dict_type = Type::typed_dict(ClassType::NonGeneric(class_literal)); - if let Some(typed_dict) = typed_dict_type.into_typed_dict() { - validate_typed_dict_constructor( - &self.context, - typed_dict, - arguments, - func.as_ref().into(), - |expr| self.expression_type(expr), - ); - } - } + if let Some(typed_dict) = callable_type.to_typed_dict_type(self.db()) { + validate_typed_dict_constructor( + &self.context, + typed_dict, + arguments, + func.as_ref().into(), + |expr| self.expression_type(expr), + ); } let mut bindings = match bindings.check_types(self.db(), &call_arguments, &tcx) { diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index 3bca7f1eca..a84983bdc7 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -807,6 +807,15 @@ impl<'db> TypeInferenceBuilder<'db, '_> { } } } + KnownInstanceType::TypedDictType(_) => { + self.infer_type_expression(slice); + + if let Some(builder) = self.context.report_lint(&NON_SUBSCRIPTABLE, subscript) { + builder.into_diagnostic(format_args!("Cannot subscript typed dict")); + } + + Type::unknown() + } KnownInstanceType::TypeAliasType(TypeAliasType::ManualPEP695(_)) => { self.infer_type_expression(slice); todo_type!("Generic manual PEP-695 type alias") diff --git a/crates/ty_python_semantic/src/types/special_form.rs b/crates/ty_python_semantic/src/types/special_form.rs index 721def0dee..7a8a19e45b 100644 --- a/crates/ty_python_semantic/src/types/special_form.rs +++ b/crates/ty_python_semantic/src/types/special_form.rs @@ -130,7 +130,7 @@ pub enum SpecialFormType { } impl SpecialFormType { - /// Return the [`KnownClass`] which this symbol is an instance of + /// Return the [`KnownClass`] which this symbol is an instance of. pub(crate) const fn class(self) -> KnownClass { match self { Self::Annotated diff --git a/crates/ty_python_semantic/src/types/type_ordering.rs b/crates/ty_python_semantic/src/types/type_ordering.rs index 3a0f1cd251..16b9b73928 100644 --- a/crates/ty_python_semantic/src/types/type_ordering.rs +++ b/crates/ty_python_semantic/src/types/type_ordering.rs @@ -212,9 +212,7 @@ pub(super) fn union_or_intersection_elements_ordering<'db>( (Type::TypeAlias(_), _) => Ordering::Less, (_, Type::TypeAlias(_)) => Ordering::Greater, - (Type::TypedDict(left), Type::TypedDict(right)) => { - left.defining_class().cmp(&right.defining_class()) - } + (Type::TypedDict(left), Type::TypedDict(right)) => left.cmp(right), (Type::TypedDict(_), _) => Ordering::Less, (_, Type::TypedDict(_)) => Ordering::Greater, diff --git a/crates/ty_python_semantic/src/types/typed_dict.rs b/crates/ty_python_semantic/src/types/typed_dict.rs index 3cfa861849..0857207179 100644 --- a/crates/ty_python_semantic/src/types/typed_dict.rs +++ b/crates/ty_python_semantic/src/types/typed_dict.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + use bitflags::bitflags; use ruff_db::diagnostic::{Annotation, Diagnostic, Span, SubDiagnostic, SubDiagnosticSeverity}; use ruff_db::parsed::parsed_module; @@ -12,6 +14,12 @@ use super::diagnostic::{ report_missing_typed_dict_key, }; use super::{ApplyTypeMappingVisitor, Type, TypeMapping, visitor}; +use crate::types::generics::GenericContext; +use crate::types::variance::TypeVarVariance; +use crate::types::{ + BoundTypeVarInstance, CallableSignature, CallableType, KnownClass, NormalizedVisitor, + Parameter, Parameters, Signature, StringLiteralType, SubclassOfType, UnionType, +}; use crate::{Db, FxOrderMap}; use ordermap::OrderSet; @@ -20,7 +28,7 @@ bitflags! { /// Used for `TypedDict` class parameters. /// Keeps track of the keyword arguments that were passed-in during class definition. /// (see https://typing.python.org/en/latest/spec/typeddict.html) - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct TypedDictParams: u8 { /// Whether keys are required by default (`total=True`) const TOTAL = 1 << 0; @@ -37,25 +45,54 @@ impl Default for TypedDictParams { /// Type that represents the set of all inhabitants (`dict` instances) that conform to /// a given `TypedDict` schema. -#[derive(Debug, Copy, Clone, PartialEq, Eq, salsa::Update, Hash, get_size2::GetSize)] -pub struct TypedDictType<'db> { +#[derive( + Copy, Clone, Debug, Eq, PartialEq, Hash, salsa::Update, PartialOrd, Ord, get_size2::GetSize, +)] +pub enum TypedDictType<'db> { /// A reference to the class (inheriting from `typing.TypedDict`) that specifies the /// schema of this `TypedDict`. - defining_class: ClassType<'db>, + FromClass(ClassType<'db>), + + /// A `TypedDict` created using the functional syntax. + Synthesized(SynthesizedTypedDictType<'db>), } impl<'db> TypedDictType<'db> { - pub(crate) fn new(defining_class: ClassType<'db>) -> Self { - Self { defining_class } + pub(crate) fn from_class(class: ClassType<'db>) -> Self { + TypedDictType::FromClass(class) } - pub(crate) fn defining_class(self) -> ClassType<'db> { - self.defining_class + /// Returns an incomplete `TypedDictType` from its items. + /// + /// This is used to instantiate a `TypedDictType` from the dictionary literal passed to a + /// `TypedDict` constructor. + pub(crate) fn from_items(db: &'db dyn Db, items: FxOrderMap>) -> Self { + TypedDictType::Synthesized(SynthesizedTypedDictType::new( + db, + None, + TypedDictParams::default(), + items, + )) } - pub(crate) fn items(self, db: &'db dyn Db) -> FxOrderMap> { - let (class_literal, specialization) = self.defining_class.class_literal(db); - class_literal.fields(db, specialization, CodeGeneratorKind::TypedDict) + pub(crate) fn items(&self, db: &'db dyn Db) -> Cow<'db, FxOrderMap>> { + match self { + TypedDictType::Synthesized(synthesized) => Cow::Borrowed(synthesized.items(db)), + TypedDictType::FromClass(class) => { + let (class_literal, specialization) = class.class_literal(db); + Cow::Owned(class_literal.fields(db, specialization, CodeGeneratorKind::TypedDict)) + } + } + } + + /// Return the meta-type of this `TypedDict` type. + pub(super) fn to_meta_type(self, db: &'db dyn Db) -> Type<'db> { + // `TypedDict` instances are instances of `dict` at runtime, but its important that we + // understand a more specific meta type in order to correctly handle `__getitem__`. + match self { + TypedDictType::FromClass(class) => SubclassOfType::from(db, class), + TypedDictType::Synthesized(_) => KnownClass::TypedDictFallback.to_class_literal(db), + } } pub(crate) fn apply_type_mapping_impl<'a>( @@ -65,12 +102,342 @@ impl<'db> TypedDictType<'db> { visitor: &ApplyTypeMappingVisitor<'db>, ) -> Self { // TODO: Materialization of gradual TypedDicts needs more logic - Self { - defining_class: self - .defining_class - .apply_type_mapping_impl(db, type_mapping, visitor), + match self { + TypedDictType::FromClass(class) => { + TypedDictType::FromClass(class.apply_type_mapping_impl(db, type_mapping, visitor)) + } + TypedDictType::Synthesized(synthesized) => TypedDictType::Synthesized( + synthesized.apply_type_mapping_impl(db, type_mapping, visitor), + ), } } + + pub(crate) fn normalized_impl( + self, + _db: &'db dyn Db, + _visitor: &NormalizedVisitor<'db>, + ) -> Self { + // TODO: Normalize typed dicts. + self + } + + /// Returns the type of a synthesized member like `__setitem__` or `__getitem__` for a `TypedDict`. + pub(crate) fn synthesized_member( + db: &'db dyn Db, + instance_ty: Type<'db>, + fields: &FxOrderMap>, + name: &str, + ) -> Option> { + match name { + "__setitem__" => { + // Add (key type, value type) overloads for all TypedDict items ("fields") that are not read-only: + let mut writeable_fields = fields + .iter() + .filter(|(_, field)| !field.is_read_only()) + .peekable(); + + if writeable_fields.peek().is_none() { + // If there are no writeable fields, synthesize a `__setitem__` that takes + // a `key` of type `Never` to signal that no keys are accepted. This leads + // to slightly more user-friendly error messages compared to returning an + // empty overload set. + return Some(Type::Callable(CallableType::new( + db, + CallableSignature::single(Signature::new( + Parameters::new([ + Parameter::positional_only(Some(Name::new_static("self"))) + .with_annotated_type(instance_ty), + Parameter::positional_only(Some(Name::new_static("key"))) + .with_annotated_type(Type::Never), + Parameter::positional_only(Some(Name::new_static("value"))) + .with_annotated_type(Type::any()), + ]), + Some(Type::none(db)), + )), + true, + ))); + } + + let overloads = writeable_fields.map(|(name, field)| { + let key_type = Type::StringLiteral(StringLiteralType::new(db, name.as_str())); + + Signature::new( + Parameters::new([ + Parameter::positional_only(Some(Name::new_static("self"))) + .with_annotated_type(instance_ty), + Parameter::positional_only(Some(Name::new_static("key"))) + .with_annotated_type(key_type), + Parameter::positional_only(Some(Name::new_static("value"))) + .with_annotated_type(field.declared_ty), + ]), + Some(Type::none(db)), + ) + }); + + Some(Type::Callable(CallableType::new( + db, + CallableSignature::from_overloads(overloads), + true, + ))) + } + "__getitem__" => { + // Add (key -> value type) overloads for all TypedDict items ("fields"): + let overloads = fields.iter().map(|(name, field)| { + let key_type = Type::StringLiteral(StringLiteralType::new(db, name.as_str())); + + Signature::new( + Parameters::new([ + Parameter::positional_only(Some(Name::new_static("self"))) + .with_annotated_type(instance_ty), + Parameter::positional_only(Some(Name::new_static("key"))) + .with_annotated_type(key_type), + ]), + Some(field.declared_ty), + ) + }); + + Some(Type::Callable(CallableType::new( + db, + CallableSignature::from_overloads(overloads), + true, + ))) + } + "get" => { + let overloads = fields + .into_iter() + .flat_map(|(name, field)| { + let key_type = + Type::StringLiteral(StringLiteralType::new(db, name.as_str())); + + // For a required key, `.get()` always returns the value type. For a non-required key, + // `.get()` returns the union of the value type and the type of the default argument + // (which defaults to `None`). + + // TODO: For now, we use two overloads here. They can be merged into a single function + // once the generics solver takes default arguments into account. + + let get_sig = Signature::new( + Parameters::new([ + Parameter::positional_only(Some(Name::new_static("self"))) + .with_annotated_type(instance_ty), + Parameter::positional_only(Some(Name::new_static("key"))) + .with_annotated_type(key_type), + ]), + Some(if field.is_required() { + field.declared_ty + } else { + UnionType::from_elements(db, [field.declared_ty, Type::none(db)]) + }), + ); + + let t_default = + BoundTypeVarInstance::synthetic(db, "T", TypeVarVariance::Covariant); + + let get_with_default_sig = Signature::new_generic( + Some(GenericContext::from_typevar_instances(db, [t_default])), + Parameters::new([ + Parameter::positional_only(Some(Name::new_static("self"))) + .with_annotated_type(instance_ty), + Parameter::positional_only(Some(Name::new_static("key"))) + .with_annotated_type(key_type), + Parameter::positional_only(Some(Name::new_static("default"))) + .with_annotated_type(Type::TypeVar(t_default)), + ]), + Some(if field.is_required() { + field.declared_ty + } else { + UnionType::from_elements( + db, + [field.declared_ty, Type::TypeVar(t_default)], + ) + }), + ); + + [get_sig, get_with_default_sig] + }) + // Fallback overloads for unknown keys + .chain(std::iter::once({ + Signature::new( + Parameters::new([ + Parameter::positional_only(Some(Name::new_static("self"))) + .with_annotated_type(instance_ty), + Parameter::positional_only(Some(Name::new_static("key"))) + .with_annotated_type(KnownClass::Str.to_instance(db)), + ]), + Some(UnionType::from_elements( + db, + [Type::unknown(), Type::none(db)], + )), + ) + })) + .chain(std::iter::once({ + let t_default = + BoundTypeVarInstance::synthetic(db, "T", TypeVarVariance::Covariant); + + Signature::new_generic( + Some(GenericContext::from_typevar_instances(db, [t_default])), + Parameters::new([ + Parameter::positional_only(Some(Name::new_static("self"))) + .with_annotated_type(instance_ty), + Parameter::positional_only(Some(Name::new_static("key"))) + .with_annotated_type(KnownClass::Str.to_instance(db)), + Parameter::positional_only(Some(Name::new_static("default"))) + .with_annotated_type(Type::TypeVar(t_default)), + ]), + Some(UnionType::from_elements( + db, + [Type::unknown(), Type::TypeVar(t_default)], + )), + ) + })); + + Some(Type::Callable(CallableType::new( + db, + CallableSignature::from_overloads(overloads), + true, + ))) + } + "pop" => { + let overloads = fields + .iter() + .filter(|(_, field)| { + // Only synthesize `pop` for fields that are not required. + !field.is_required() + }) + .flat_map(|(name, field)| { + let key_type = + Type::StringLiteral(StringLiteralType::new(db, name.as_str())); + + // TODO: Similar to above: consider merging these two overloads into one + + // `.pop()` without default + let pop_sig = Signature::new( + Parameters::new([ + Parameter::positional_only(Some(Name::new_static("self"))) + .with_annotated_type(instance_ty), + Parameter::positional_only(Some(Name::new_static("key"))) + .with_annotated_type(key_type), + ]), + Some(field.declared_ty), + ); + + // `.pop()` with a default value + let t_default = + BoundTypeVarInstance::synthetic(db, "T", TypeVarVariance::Covariant); + + let pop_with_default_sig = Signature::new_generic( + Some(GenericContext::from_typevar_instances(db, [t_default])), + Parameters::new([ + Parameter::positional_only(Some(Name::new_static("self"))) + .with_annotated_type(instance_ty), + Parameter::positional_only(Some(Name::new_static("key"))) + .with_annotated_type(key_type), + Parameter::positional_only(Some(Name::new_static("default"))) + .with_annotated_type(Type::TypeVar(t_default)), + ]), + Some(UnionType::from_elements( + db, + [field.declared_ty, Type::TypeVar(t_default)], + )), + ); + + [pop_sig, pop_with_default_sig] + }); + + Some(Type::Callable(CallableType::new( + db, + CallableSignature::from_overloads(overloads), + true, + ))) + } + "setdefault" => { + let overloads = fields.iter().map(|(name, field)| { + let key_type = Type::StringLiteral(StringLiteralType::new(db, name.as_str())); + + // `setdefault` always returns the field type + Signature::new( + Parameters::new([ + Parameter::positional_only(Some(Name::new_static("self"))) + .with_annotated_type(instance_ty), + Parameter::positional_only(Some(Name::new_static("key"))) + .with_annotated_type(key_type), + Parameter::positional_only(Some(Name::new_static("default"))) + .with_annotated_type(field.declared_ty), + ]), + Some(field.declared_ty), + ) + }); + + Some(Type::Callable(CallableType::new( + db, + CallableSignature::from_overloads(overloads), + true, + ))) + } + "update" => { + // TODO: synthesize a set of overloads with precise types + let signature = Signature::new( + Parameters::new([ + Parameter::positional_only(Some(Name::new_static("self"))) + .with_annotated_type(instance_ty), + Parameter::variadic(Name::new_static("args")), + Parameter::keyword_variadic(Name::new_static("kwargs")), + ]), + Some(Type::none(db)), + ); + + Some(CallableType::function_like(db, signature)) + } + _ => None, + } + } +} + +#[salsa::interned(debug, heap_size=SynthesizedTypedDictType::heap_size)] +#[derive(PartialOrd, Ord)] +pub struct SynthesizedTypedDictType<'db> { + // The dictionary literal passed to the `TypedDict` constructor is inferred as + // a nameless `SynthesizedTypedDictType`. + pub(crate) name: Option, + + pub(crate) params: TypedDictParams, + + #[returns(ref)] + pub(crate) items: FxOrderMap>, +} + +// The Salsa heap is tracked separately. +impl get_size2::GetSize for SynthesizedTypedDictType<'_> {} + +impl<'db> SynthesizedTypedDictType<'db> { + pub(super) fn apply_type_mapping_impl<'a>( + self, + db: &'db dyn Db, + type_mapping: &TypeMapping<'a, 'db>, + visitor: &ApplyTypeMappingVisitor<'db>, + ) -> Self { + let items = self + .items(db) + .iter() + .map(|(name, field)| { + let field = field + .clone() + .apply_type_mapping_impl(db, type_mapping, visitor); + + (name.clone(), field) + }) + .collect::>(); + + SynthesizedTypedDictType::new(db, self.name(db), self.params(db), items) + } + + fn heap_size( + (name, params, items): &(Option, TypedDictParams, FxOrderMap>), + ) -> usize { + ruff_memory_usage::heap_size(name) + + ruff_memory_usage::heap_size(params) + + ruff_memory_usage::order_map_heap_size(items) + } } pub(crate) fn walk_typed_dict_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>( @@ -78,7 +445,14 @@ pub(crate) fn walk_typed_dict_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>( typed_dict: TypedDictType<'db>, visitor: &V, ) { - visitor.visit_type(db, typed_dict.defining_class.into()); + match typed_dict { + TypedDictType::FromClass(class) => visitor.visit_type(db, class.into()), + TypedDictType::Synthesized(synthesized) => { + for (_, item) in synthesized.items(db) { + visitor.visit_type(db, item.declared_ty); + } + } + } } pub(super) fn typed_dict_params_from_class_def(class_stmt: &StmtClassDef) -> TypedDictParams {