From b6231895602b57b289d0bce732df9b7aed0e70ba Mon Sep 17 00:00:00 2001 From: Dhruv Manilawala Date: Fri, 5 Dec 2025 22:00:06 +0530 Subject: [PATCH] [ty] Complete support for `ParamSpec` (#21445) ## Summary Closes: https://github.com/astral-sh/ty/issues/157 This PR adds support for the following capabilities involving a `ParamSpec` type variable: - Representing `P.args` and `P.kwargs` in the type system - Matching against a callable containing `P` to create a type mapping - Specializing `P` against the stored parameters The value of a `ParamSpec` type variable is being represented using `CallableType` with a `CallableTypeKind::ParamSpecValue` variant. This `CallableTypeKind` is expanded from the existing `is_function_like` boolean flag. An `enum` is used as these variants are mutually exclusive. For context, an initial iteration made an attempt to expand the `Specialization` to use `TypeOrParameters` enum that represents that a type variable can specialize into either a `Type` or `Parameters` but that increased the complexity of the code as all downstream usages would need to handle both the variants appropriately. Additionally, we'd have also need to establish an invariant that a regular type variable always maps to a `Type` while a paramspec type variable always maps to a `Parameters`. I've intentionally left out checking and raising diagnostics when the `ParamSpec` type variable and it's components are not being used correctly to avoid scope increase and it can easily be done as a follow-up. This would also include the scoping rules which I don't think a regular type variable implements either. ## Test Plan Add new mdtest cases and update existing test cases. Ran this branch on pyx, no new diagnostics. ### Ecosystem analysis There's a case where in an annotated assignment like: ```py type CustomType[P] = Callable[...] def value[**P](...): ... def another[**P](...): target: CustomType[P] = value ``` The type of `value` is a callable and it has a paramspec that's bound to `value`, `CustomType` is a type alias that's a callable and `P` that's used in it's specialization is bound to `another`. Now, ty infers the type of `target` same as `value` and does not use the declared type `CustomType[P]`. [This is the assignment](https://github.com/mikeshardmind/async-utils/blob/0980b9d9ab2bc7a24777684a884f4ea96cbbe5f9/src/async_utils/gen_transform.py#L108) that I'm referring to which then leads to error in downstream usage. Pyright and mypy does seem to use the declared type. There are multiple diagnostics in `dd-trace-py` that requires support for `cls`. I'm seeing `Divergent` type for an example like which ~~I'm not sure why, I'll look into it tomorrow~~ is because of a cycle as mentioned in https://github.com/astral-sh/ty/issues/1729#issuecomment-3612279974: ```py from typing import Callable def decorator[**P](c: Callable[P, int]) -> Callable[P, str]: ... @decorator def func(a: int) -> int: ... # ((a: int) -> str) | ((a: Divergent) -> str) reveal_type(func) ``` I ~~need to look into why are the parameters not being specialized through multiple decorators in the following code~~ think this is also because of the cycle mentioned in https://github.com/astral-sh/ty/issues/1729#issuecomment-3612279974 and the fact that we don't support `staticmethod` properly: ```py from contextlib import contextmanager class Foo: @staticmethod @contextmanager def method(x: int): yield foo = Foo() # ty: Revealed type: `() -> _GeneratorContextManager[Unknown, None, None]` [revealed-type] reveal_type(foo.method) ``` There's some issue related to `Protocol` that are generic over a `ParamSpec` in `starlette` which might be related to https://github.com/astral-sh/ty/issues/1635 but I'm not sure. Here's a minimal example to reproduce:
Code snippet:

```py from collections.abc import Awaitable, Callable, MutableMapping from typing import Any, Callable, ParamSpec, Protocol P = ParamSpec("P") Scope = MutableMapping[str, Any] Message = MutableMapping[str, Any] Receive = Callable[[], Awaitable[Message]] Send = Callable[[Message], Awaitable[None]] ASGIApp = Callable[[Scope, Receive, Send], Awaitable[None]] _Scope = Any _Receive = Callable[[], Awaitable[Any]] _Send = Callable[[Any], Awaitable[None]] # Since `starlette.types.ASGIApp` type differs from `ASGIApplication` from `asgiref` # we need to define a more permissive version of ASGIApp that doesn't cause type errors. _ASGIApp = Callable[[_Scope, _Receive, _Send], Awaitable[None]] class _MiddlewareFactory(Protocol[P]): def __call__( self, app: _ASGIApp, *args: P.args, **kwargs: P.kwargs ) -> _ASGIApp: ... class Middleware: def __init__( self, factory: _MiddlewareFactory[P], *args: P.args, **kwargs: P.kwargs ) -> None: self.factory = factory self.args = args self.kwargs = kwargs class ServerErrorMiddleware: def __init__( self, app: ASGIApp, value: int | None = None, flag: bool = False, ) -> None: self.app = app self.value = value self.flag = flag async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: ... # ty: Argument to bound method `__init__` is incorrect: Expected `_MiddlewareFactory[(...)]`, found `` [invalid-argument-type] Middleware(ServerErrorMiddleware, value=500, flag=True) ```

### Conformance analysis > ```diff > -constructors_callable.py:36:13: info[revealed-type] Revealed type: `(...) -> Unknown` > +constructors_callable.py:36:13: info[revealed-type] Revealed type: `(x: int) -> Unknown` > ``` Requires return type inference i.e., https://github.com/astral-sh/ruff/pull/21551 > ```diff > +constructors_callable.py:194:16: error[invalid-argument-type] Argument is incorrect: Expected `list[T@__init__]`, found `list[Unknown | str]` > +constructors_callable.py:194:22: error[invalid-argument-type] Argument is incorrect: Expected `list[T@__init__]`, found `list[Unknown | str]` > +constructors_callable.py:195:4: error[invalid-argument-type] Argument is incorrect: Expected `list[T@__init__]`, found `list[Unknown | int]` > +constructors_callable.py:195:9: error[invalid-argument-type] Argument is incorrect: Expected `list[T@__init__]`, found `list[Unknown | str]` > ``` I might need to look into why this is happening... > ```diff > +generics_defaults.py:79:1: error[type-assertion-failure] Type `type[Class_ParamSpec[(str, int, /)]]` does not match asserted type `` > ``` which is on the following code ```py DefaultP = ParamSpec("DefaultP", default=[str, int]) class Class_ParamSpec(Generic[DefaultP]): ... assert_type(Class_ParamSpec, type[Class_ParamSpec[str, int]]) ``` It's occurring because there's no equivalence relationship defined between `ClassLiteral` and `KnownInstanceType::TypeGenericAlias` which is what these types are. Everything else looks good to me! --- crates/ty_ide/src/hover.rs | 16 +- .../resources/mdtest/annotations/callable.md | 11 +- .../annotations/unsupported_special_forms.md | 8 +- .../mdtest/generics/legacy/paramspec.md | 268 ++++++++ .../mdtest/generics/pep695/aliases.md | 8 +- .../mdtest/generics/pep695/classes.md | 8 +- .../mdtest/generics/pep695/paramspec.md | 585 ++++++++++++++++++ .../resources/mdtest/implicit_type_aliases.md | 13 +- .../type_properties/is_assignable_to.md | 32 + .../resources/mdtest/with/async.md | 2 +- crates/ty_python_semantic/src/types.rs | 347 +++++++++-- .../src/types/call/arguments.rs | 8 + .../ty_python_semantic/src/types/call/bind.rs | 388 +++++++++--- crates/ty_python_semantic/src/types/class.rs | 44 +- .../ty_python_semantic/src/types/display.rs | 222 ++++--- .../ty_python_semantic/src/types/function.rs | 6 +- .../ty_python_semantic/src/types/generics.rs | 96 ++- .../src/types/infer/builder.rs | 440 +++++++++---- .../infer/builder/annotation_expression.rs | 16 +- .../types/infer/builder/type_expression.rs | 43 +- .../src/types/protocol_class.rs | 8 +- .../src/types/signatures.rs | 280 ++++++++- 22 files changed, 2410 insertions(+), 439 deletions(-) diff --git a/crates/ty_ide/src/hover.rs b/crates/ty_ide/src/hover.rs index 63b67bc864..8f9add508a 100644 --- a/crates/ty_ide/src/hover.rs +++ b/crates/ty_ide/src/hover.rs @@ -2143,15 +2143,13 @@ def function(): "#, ); + // TODO: This should just be `**AB@Alias2 ()` + // https://github.com/astral-sh/ty/issues/1581 assert_snapshot!(test.hover(), @r" - ( - ... - ) -> tuple[typing.ParamSpec] + (**AB@Alias2) -> tuple[AB@Alias2] --------------------------------------------- ```python - ( - ... - ) -> tuple[typing.ParamSpec] + (**AB@Alias2) -> tuple[AB@Alias2] ``` --------------------------------------------- info[hover]: Hovered content is @@ -2292,12 +2290,12 @@ def function(): "#, ); - // TODO: This should be `P@Alias ()` + // TODO: Should this be constravariant instead? assert_snapshot!(test.hover(), @r" - typing.ParamSpec + P@Alias (bivariant) --------------------------------------------- ```python - typing.ParamSpec + P@Alias (bivariant) ``` --------------------------------------------- info[hover]: Hovered content is diff --git a/crates/ty_python_semantic/resources/mdtest/annotations/callable.md b/crates/ty_python_semantic/resources/mdtest/annotations/callable.md index e7e55f7a44..728566d30e 100644 --- a/crates/ty_python_semantic/resources/mdtest/annotations/callable.md +++ b/crates/ty_python_semantic/resources/mdtest/annotations/callable.md @@ -307,12 +307,10 @@ Using a `ParamSpec` in a `Callable` annotation: from typing_extensions import Callable def _[**P1](c: Callable[P1, int]): - # TODO: Should reveal `ParamSpecArgs` and `ParamSpecKwargs` - reveal_type(P1.args) # revealed: @Todo(ParamSpecArgs / ParamSpecKwargs) - reveal_type(P1.kwargs) # revealed: @Todo(ParamSpecArgs / ParamSpecKwargs) + reveal_type(P1.args) # revealed: P1@_.args + reveal_type(P1.kwargs) # revealed: P1@_.kwargs - # TODO: Signature should be (**P1) -> int - reveal_type(c) # revealed: (...) -> int + reveal_type(c) # revealed: (**P1@_) -> int ``` And, using the legacy syntax: @@ -322,9 +320,8 @@ from typing_extensions import ParamSpec P2 = ParamSpec("P2") -# TODO: argument list should not be `...` (requires `ParamSpec` support) def _(c: Callable[P2, int]): - reveal_type(c) # revealed: (...) -> int + reveal_type(c) # revealed: (**P2@_) -> int ``` ## Using `typing.Unpack` diff --git a/crates/ty_python_semantic/resources/mdtest/annotations/unsupported_special_forms.md b/crates/ty_python_semantic/resources/mdtest/annotations/unsupported_special_forms.md index cfc485cb53..18ebd03682 100644 --- a/crates/ty_python_semantic/resources/mdtest/annotations/unsupported_special_forms.md +++ b/crates/ty_python_semantic/resources/mdtest/annotations/unsupported_special_forms.md @@ -18,9 +18,8 @@ def f(*args: Unpack[Ts]) -> tuple[Unpack[Ts]]: def g() -> TypeGuard[int]: ... def i(callback: Callable[Concatenate[int, P], R_co], *args: P.args, **kwargs: P.kwargs) -> R_co: - # TODO: Should reveal a type representing `P.args` and `P.kwargs` - reveal_type(args) # revealed: tuple[@Todo(ParamSpecArgs / ParamSpecKwargs), ...] - reveal_type(kwargs) # revealed: dict[str, @Todo(ParamSpecArgs / ParamSpecKwargs)] + reveal_type(args) # revealed: P@i.args + reveal_type(kwargs) # revealed: P@i.kwargs return callback(42, *args, **kwargs) class Foo: @@ -65,8 +64,9 @@ def _( reveal_type(c) # revealed: Unknown reveal_type(d) # revealed: Unknown + # error: [invalid-type-form] "Variable of type `ParamSpec` is not allowed in a type expression" def foo(a_: e) -> None: - reveal_type(a_) # revealed: @Todo(Support for `typing.ParamSpec`) + reveal_type(a_) # revealed: Unknown ``` ## Inheritance diff --git a/crates/ty_python_semantic/resources/mdtest/generics/legacy/paramspec.md b/crates/ty_python_semantic/resources/mdtest/generics/legacy/paramspec.md index 2ce9f14852..c4764c3886 100644 --- a/crates/ty_python_semantic/resources/mdtest/generics/legacy/paramspec.md +++ b/crates/ty_python_semantic/resources/mdtest/generics/legacy/paramspec.md @@ -115,3 +115,271 @@ P = ParamSpec("P", default=[A, B]) class A: ... class B: ... ``` + +## Validating `ParamSpec` usage + +In type annotations, `ParamSpec` is only valid as the first element to `Callable`, the final element +to `Concatenate`, or as a type parameter to `Protocol` or `Generic`. + +```py +from typing import ParamSpec, Callable, Concatenate, Protocol, Generic + +P = ParamSpec("P") + +class ValidProtocol(Protocol[P]): + def method(self, c: Callable[P, int]) -> None: ... + +class ValidGeneric(Generic[P]): + def method(self, c: Callable[P, int]) -> None: ... + +def valid( + a1: Callable[P, int], + a2: Callable[Concatenate[int, P], int], +) -> None: ... +def invalid( + # TODO: error + a1: P, + # TODO: error + a2: list[P], + # TODO: error + a3: Callable[[P], int], + # TODO: error + a4: Callable[..., P], + # TODO: error + a5: Callable[Concatenate[P, ...], int], +) -> None: ... +``` + +## Validating `P.args` and `P.kwargs` usage + +The components of `ParamSpec` i.e., `P.args` and `P.kwargs` are only valid when used as the +annotated types of `*args` and `**kwargs` respectively. + +```py +from typing import Generic, Callable, ParamSpec + +P = ParamSpec("P") + +def foo1(c: Callable[P, int]) -> None: + def nested1(*args: P.args, **kwargs: P.kwargs) -> None: ... + def nested2( + # error: [invalid-type-form] "`P.kwargs` is valid only in `**kwargs` annotation: Did you mean `P.args`?" + *args: P.kwargs, + # error: [invalid-type-form] "`P.args` is valid only in `*args` annotation: Did you mean `P.kwargs`?" + **kwargs: P.args, + ) -> None: ... + + # TODO: error + def nested3(*args: P.args) -> None: ... + + # TODO: error + def nested4(**kwargs: P.kwargs) -> None: ... + + # TODO: error + def nested5(*args: P.args, x: int, **kwargs: P.kwargs) -> None: ... + +# TODO: error +def bar1(*args: P.args, **kwargs: P.kwargs) -> None: + pass + +class Foo1: + # TODO: error + def method(self, *args: P.args, **kwargs: P.kwargs) -> None: ... +``` + +And, they need to be used together. + +```py +def foo2(c: Callable[P, int]) -> None: + # TODO: error + def nested1(*args: P.args) -> None: ... + + # TODO: error + def nested2(**kwargs: P.kwargs) -> None: ... + +class Foo2: + # TODO: error + args: P.args + + # TODO: error + kwargs: P.kwargs +``` + +The name of these parameters does not need to be `args` or `kwargs`, it's the annotated type to the +respective variadic parameter that matters. + +```py +class Foo3(Generic[P]): + def method1(self, *paramspec_args: P.args, **paramspec_kwargs: P.kwargs) -> None: ... + def method2( + self, + # error: [invalid-type-form] "`P.kwargs` is valid only in `**kwargs` annotation: Did you mean `P.args`?" + *paramspec_args: P.kwargs, + # error: [invalid-type-form] "`P.args` is valid only in `*args` annotation: Did you mean `P.kwargs`?" + **paramspec_kwargs: P.args, + ) -> None: ... +``` + +## Specializing generic classes explicitly + +```py +from typing import Any, Generic, ParamSpec, Callable, TypeVar + +P1 = ParamSpec("P1") +P2 = ParamSpec("P2") +T1 = TypeVar("T1") + +class OnlyParamSpec(Generic[P1]): + attr: Callable[P1, None] + +class TwoParamSpec(Generic[P1, P2]): + attr1: Callable[P1, None] + attr2: Callable[P2, None] + +class TypeVarAndParamSpec(Generic[T1, P1]): + attr: Callable[P1, T1] +``` + +Explicit specialization of a generic class involving `ParamSpec` is done by providing either a list +of types, `...`, or another in-scope `ParamSpec`. + +```py +reveal_type(OnlyParamSpec[[int, str]]().attr) # revealed: (int, str, /) -> None +reveal_type(OnlyParamSpec[...]().attr) # revealed: (...) -> None + +def func(c: Callable[P2, None]): + reveal_type(OnlyParamSpec[P2]().attr) # revealed: (**P2@func) -> None + +# TODO: error: paramspec is unbound +reveal_type(OnlyParamSpec[P2]().attr) # revealed: (...) -> None +``` + +The square brackets can be omitted when `ParamSpec` is the only type variable + +```py +reveal_type(OnlyParamSpec[int, str]().attr) # revealed: (int, str, /) -> None +reveal_type(OnlyParamSpec[int,]().attr) # revealed: (int, /) -> None + +# Even when there is only one element +reveal_type(OnlyParamSpec[Any]().attr) # revealed: (Any, /) -> None +reveal_type(OnlyParamSpec[object]().attr) # revealed: (object, /) -> None +reveal_type(OnlyParamSpec[int]().attr) # revealed: (int, /) -> None +``` + +But, they cannot be omitted when there are multiple type variables. + +```py +reveal_type(TypeVarAndParamSpec[int, [int, str]]().attr) # revealed: (int, str, /) -> int +reveal_type(TypeVarAndParamSpec[int, [str]]().attr) # revealed: (str, /) -> int +reveal_type(TypeVarAndParamSpec[int, ...]().attr) # revealed: (...) -> int + +# TODO: We could still specialize for `T1` as the type is valid which would reveal `(...) -> int` +# TODO: error: paramspec is unbound +reveal_type(TypeVarAndParamSpec[int, P2]().attr) # revealed: (...) -> Unknown +# error: [invalid-type-arguments] "Type argument for `ParamSpec` must be either a list of types, `ParamSpec`, `Concatenate`, or `...`" +reveal_type(TypeVarAndParamSpec[int, int]().attr) # revealed: (...) -> Unknown +``` + +Nor can they be omitted when there are more than one `ParamSpec`s. + +```py +p = TwoParamSpec[[int, str], [int]]() +reveal_type(p.attr1) # revealed: (int, str, /) -> None +reveal_type(p.attr2) # revealed: (int, /) -> None + +# error: [invalid-type-arguments] +# error: [invalid-type-arguments] +TwoParamSpec[int, str] +``` + +Specializing `ParamSpec` type variable using `typing.Any` isn't explicitly allowed by the spec but +both mypy and Pyright allow this and there are usages of this in the wild e.g., +`staticmethod[Any, Any]`. + +```py +reveal_type(TypeVarAndParamSpec[int, Any]().attr) # revealed: (...) -> int +``` + +## Specialization when defaults are involved + +```toml +[environment] +python-version = "3.13" +``` + +```py +from typing import Any, Generic, ParamSpec, Callable, TypeVar + +P = ParamSpec("P") +PList = ParamSpec("PList", default=[int, str]) +PEllipsis = ParamSpec("PEllipsis", default=...) +PAnother = ParamSpec("PAnother", default=P) +PAnotherWithDefault = ParamSpec("PAnotherWithDefault", default=PList) +``` + +```py +class ParamSpecWithDefault1(Generic[PList]): + attr: Callable[PList, None] + +reveal_type(ParamSpecWithDefault1().attr) # revealed: (int, str, /) -> None +reveal_type(ParamSpecWithDefault1[[int]]().attr) # revealed: (int, /) -> None +``` + +```py +class ParamSpecWithDefault2(Generic[PEllipsis]): + attr: Callable[PEllipsis, None] + +reveal_type(ParamSpecWithDefault2().attr) # revealed: (...) -> None +reveal_type(ParamSpecWithDefault2[[int, str]]().attr) # revealed: (int, str, /) -> None +``` + +```py +class ParamSpecWithDefault3(Generic[P, PAnother]): + attr1: Callable[P, None] + attr2: Callable[PAnother, None] + +# `P` hasn't been specialized, so it defaults to `Unknown` gradual form +p1 = ParamSpecWithDefault3() +reveal_type(p1.attr1) # revealed: (...) -> None +reveal_type(p1.attr2) # revealed: (...) -> None + +p2 = ParamSpecWithDefault3[[int, str]]() +reveal_type(p2.attr1) # revealed: (int, str, /) -> None +reveal_type(p2.attr2) # revealed: (int, str, /) -> None + +p3 = ParamSpecWithDefault3[[int], [str]]() +reveal_type(p3.attr1) # revealed: (int, /) -> None +reveal_type(p3.attr2) # revealed: (str, /) -> None + +class ParamSpecWithDefault4(Generic[PList, PAnotherWithDefault]): + attr1: Callable[PList, None] + attr2: Callable[PAnotherWithDefault, None] + +p1 = ParamSpecWithDefault4() +reveal_type(p1.attr1) # revealed: (int, str, /) -> None +reveal_type(p1.attr2) # revealed: (int, str, /) -> None + +p2 = ParamSpecWithDefault4[[int]]() +reveal_type(p2.attr1) # revealed: (int, /) -> None +reveal_type(p2.attr2) # revealed: (int, /) -> None + +p3 = ParamSpecWithDefault4[[int], [str]]() +reveal_type(p3.attr1) # revealed: (int, /) -> None +reveal_type(p3.attr2) # revealed: (str, /) -> None + +# TODO: error +# Un-ordered type variables as the default of `PAnother` is `P` +class ParamSpecWithDefault5(Generic[PAnother, P]): + attr: Callable[PAnother, None] + +# TODO: error +# PAnother has default as P (another ParamSpec) which is not in scope +class ParamSpecWithDefault6(Generic[PAnother]): + attr: Callable[PAnother, None] +``` + +## Semantics + +The semantics of `ParamSpec` are described in +[the PEP 695 `ParamSpec` document](./../pep695/paramspec.md) to avoid duplication unless there are +any behavior specific to the legacy `ParamSpec` implementation. diff --git a/crates/ty_python_semantic/resources/mdtest/generics/pep695/aliases.md b/crates/ty_python_semantic/resources/mdtest/generics/pep695/aliases.md index e10febeaeb..490ae01fc6 100644 --- a/crates/ty_python_semantic/resources/mdtest/generics/pep695/aliases.md +++ b/crates/ty_python_semantic/resources/mdtest/generics/pep695/aliases.md @@ -25,11 +25,11 @@ reveal_type(generic_context(SingleTypevar)) # revealed: ty_extensions.GenericContext[T@MultipleTypevars, S@MultipleTypevars] reveal_type(generic_context(MultipleTypevars)) -# TODO: support `ParamSpec`/`TypeVarTuple` properly -# (these should include the `ParamSpec`s and `TypeVarTuple`s in their generic contexts) -# revealed: ty_extensions.GenericContext[] +# TODO: support `TypeVarTuple` properly +# (these should include the `TypeVarTuple`s in their generic contexts) +# revealed: ty_extensions.GenericContext[P@SingleParamSpec] reveal_type(generic_context(SingleParamSpec)) -# revealed: ty_extensions.GenericContext[T@TypeVarAndParamSpec] +# revealed: ty_extensions.GenericContext[T@TypeVarAndParamSpec, P@TypeVarAndParamSpec] reveal_type(generic_context(TypeVarAndParamSpec)) # revealed: ty_extensions.GenericContext[] reveal_type(generic_context(SingleTypeVarTuple)) diff --git a/crates/ty_python_semantic/resources/mdtest/generics/pep695/classes.md b/crates/ty_python_semantic/resources/mdtest/generics/pep695/classes.md index dbb249b45e..71e05171e8 100644 --- a/crates/ty_python_semantic/resources/mdtest/generics/pep695/classes.md +++ b/crates/ty_python_semantic/resources/mdtest/generics/pep695/classes.md @@ -25,11 +25,11 @@ reveal_type(generic_context(SingleTypevar)) # revealed: ty_extensions.GenericContext[T@MultipleTypevars, S@MultipleTypevars] reveal_type(generic_context(MultipleTypevars)) -# TODO: support `ParamSpec`/`TypeVarTuple` properly -# (these should include the `ParamSpec`s and `TypeVarTuple`s in their generic contexts) -# revealed: ty_extensions.GenericContext[] +# TODO: support `TypeVarTuple` properly +# (these should include the `TypeVarTuple`s in their generic contexts) +# revealed: ty_extensions.GenericContext[P@SingleParamSpec] reveal_type(generic_context(SingleParamSpec)) -# revealed: ty_extensions.GenericContext[T@TypeVarAndParamSpec] +# revealed: ty_extensions.GenericContext[T@TypeVarAndParamSpec, P@TypeVarAndParamSpec] reveal_type(generic_context(TypeVarAndParamSpec)) # revealed: ty_extensions.GenericContext[] reveal_type(generic_context(SingleTypeVarTuple)) diff --git a/crates/ty_python_semantic/resources/mdtest/generics/pep695/paramspec.md b/crates/ty_python_semantic/resources/mdtest/generics/pep695/paramspec.md index 62b50b05ef..6483428bb3 100644 --- a/crates/ty_python_semantic/resources/mdtest/generics/pep695/paramspec.md +++ b/crates/ty_python_semantic/resources/mdtest/generics/pep695/paramspec.md @@ -62,3 +62,588 @@ Other values are invalid. def foo[**P = int]() -> None: pass ``` + +## Validating `ParamSpec` usage + +`ParamSpec` is only valid as the first element to `Callable` or the final element to `Concatenate`. + +```py +from typing import ParamSpec, Callable, Concatenate + +def valid[**P]( + a1: Callable[P, int], + a2: Callable[Concatenate[int, P], int], +) -> None: ... +def invalid[**P]( + # TODO: error + a1: P, + # TODO: error + a2: list[P], + # TODO: error + a3: Callable[[P], int], + # TODO: error + a4: Callable[..., P], + # TODO: error + a5: Callable[Concatenate[P, ...], int], +) -> None: ... +``` + +## Validating `P.args` and `P.kwargs` usage + +The components of `ParamSpec` i.e., `P.args` and `P.kwargs` are only valid when used as the +annotated types of `*args` and `**kwargs` respectively. + +```py +from typing import Callable + +def foo[**P](c: Callable[P, int]) -> None: + def nested1(*args: P.args, **kwargs: P.kwargs) -> None: ... + + # error: [invalid-type-form] "`P.kwargs` is valid only in `**kwargs` annotation: Did you mean `P.args`?" + # error: [invalid-type-form] "`P.args` is valid only in `*args` annotation: Did you mean `P.kwargs`?" + def nested2(*args: P.kwargs, **kwargs: P.args) -> None: ... + + # TODO: error + def nested3(*args: P.args) -> None: ... + + # TODO: error + def nested4(**kwargs: P.kwargs) -> None: ... + + # TODO: error + def nested5(*args: P.args, x: int, **kwargs: P.kwargs) -> None: ... +``` + +And, they need to be used together. + +```py +def foo[**P](c: Callable[P, int]) -> None: + # TODO: error + def nested1(*args: P.args) -> None: ... + + # TODO: error + def nested2(**kwargs: P.kwargs) -> None: ... + +class Foo[**P]: + # TODO: error + args: P.args + + # TODO: error + kwargs: P.kwargs +``` + +The name of these parameters does not need to be `args` or `kwargs`, it's the annotated type to the +respective variadic parameter that matters. + +```py +class Foo3[**P]: + def method1(self, *paramspec_args: P.args, **paramspec_kwargs: P.kwargs) -> None: ... + def method2( + self, + # error: [invalid-type-form] "`P.kwargs` is valid only in `**kwargs` annotation: Did you mean `P.args`?" + *paramspec_args: P.kwargs, + # error: [invalid-type-form] "`P.args` is valid only in `*args` annotation: Did you mean `P.kwargs`?" + **paramspec_kwargs: P.args, + ) -> None: ... +``` + +It isn't allowed to annotate an instance attribute either: + +```py +class Foo4[**P]: + def __init__(self, fn: Callable[P, int], *args: P.args, **kwargs: P.kwargs) -> None: + self.fn = fn + # TODO: error + self.args: P.args = args + # TODO: error + self.kwargs: P.kwargs = kwargs +``` + +## Semantics of `P.args` and `P.kwargs` + +The type of `args` and `kwargs` inside the function is `P.args` and `P.kwargs` respectively instead +of `tuple[P.args, ...]` and `dict[str, P.kwargs]`. + +### Passing `*args` and `**kwargs` to a callable + +```py +from typing import Callable + +def f[**P](func: Callable[P, int]) -> Callable[P, None]: + def wrapper(*args: P.args, **kwargs: P.kwargs) -> None: + reveal_type(args) # revealed: P@f.args + reveal_type(kwargs) # revealed: P@f.kwargs + reveal_type(func(*args, **kwargs)) # revealed: int + + # error: [invalid-argument-type] "Argument is incorrect: Expected `P@f.args`, found `P@f.kwargs`" + # error: [invalid-argument-type] "Argument is incorrect: Expected `P@f.kwargs`, found `P@f.args`" + reveal_type(func(*kwargs, **args)) # revealed: int + + # error: [invalid-argument-type] "Argument is incorrect: Expected `P@f.args`, found `P@f.kwargs`" + reveal_type(func(args, kwargs)) # revealed: int + + # Both parameters are required + # TODO: error + reveal_type(func()) # revealed: int + reveal_type(func(*args)) # revealed: int + reveal_type(func(**kwargs)) # revealed: int + return wrapper +``` + +### Operations on `P.args` and `P.kwargs` + +The type of `P.args` and `P.kwargs` behave like a `tuple` and `dict` respectively. Internally, they +are represented as a type variable that has an upper bound of `tuple[object, ...]` and +`Top[dict[str, Any]]` respectively. + +```py +from typing import Callable, Any + +def f[**P](func: Callable[P, int], *args: P.args, **kwargs: P.kwargs) -> None: + reveal_type(args + ("extra",)) # revealed: tuple[object, ...] + reveal_type(args + (1, 2, 3)) # revealed: tuple[object, ...] + reveal_type(args[0]) # revealed: object + + reveal_type("key" in kwargs) # revealed: bool + reveal_type(kwargs.get("key")) # revealed: object + reveal_type(kwargs["key"]) # revealed: object +``` + +## Specializing generic classes explicitly + +```py +from typing import Any, Callable, ParamSpec + +class OnlyParamSpec[**P1]: + attr: Callable[P1, None] + +class TwoParamSpec[**P1, **P2]: + attr1: Callable[P1, None] + attr2: Callable[P2, None] + +class TypeVarAndParamSpec[T1, **P1]: + attr: Callable[P1, T1] +``` + +Explicit specialization of a generic class involving `ParamSpec` is done by providing either a list +of types, `...`, or another in-scope `ParamSpec`. + +```py +reveal_type(OnlyParamSpec[[int, str]]().attr) # revealed: (int, str, /) -> None +reveal_type(OnlyParamSpec[...]().attr) # revealed: (...) -> None + +def func[**P2](c: Callable[P2, None]): + reveal_type(OnlyParamSpec[P2]().attr) # revealed: (**P2@func) -> None + +P2 = ParamSpec("P2") + +# TODO: error: paramspec is unbound +reveal_type(OnlyParamSpec[P2]().attr) # revealed: (...) -> None +``` + +The square brackets can be omitted when `ParamSpec` is the only type variable + +```py +reveal_type(OnlyParamSpec[int, str]().attr) # revealed: (int, str, /) -> None +reveal_type(OnlyParamSpec[int,]().attr) # revealed: (int, /) -> None + +# Even when there is only one element +reveal_type(OnlyParamSpec[Any]().attr) # revealed: (Any, /) -> None +reveal_type(OnlyParamSpec[object]().attr) # revealed: (object, /) -> None +reveal_type(OnlyParamSpec[int]().attr) # revealed: (int, /) -> None +``` + +But, they cannot be omitted when there are multiple type variables. + +```py +reveal_type(TypeVarAndParamSpec[int, [int, str]]().attr) # revealed: (int, str, /) -> int +reveal_type(TypeVarAndParamSpec[int, [str]]().attr) # revealed: (str, /) -> int +reveal_type(TypeVarAndParamSpec[int, ...]().attr) # revealed: (...) -> int + +# TODO: error: paramspec is unbound +reveal_type(TypeVarAndParamSpec[int, P2]().attr) # revealed: (...) -> Unknown +# error: [invalid-type-arguments] +reveal_type(TypeVarAndParamSpec[int, int]().attr) # revealed: (...) -> Unknown +``` + +Nor can they be omitted when there are more than one `ParamSpec`. + +```py +p = TwoParamSpec[[int, str], [int]]() +reveal_type(p.attr1) # revealed: (int, str, /) -> None +reveal_type(p.attr2) # revealed: (int, /) -> None + +# error: [invalid-type-arguments] "Type argument for `ParamSpec` must be either a list of types, `ParamSpec`, `Concatenate`, or `...`" +# error: [invalid-type-arguments] "Type argument for `ParamSpec` must be either a list of types, `ParamSpec`, `Concatenate`, or `...`" +TwoParamSpec[int, str] +``` + +Specializing `ParamSpec` type variable using `typing.Any` isn't explicitly allowed by the spec but +both mypy and Pyright allow this and there are usages of this in the wild e.g., +`staticmethod[Any, Any]`. + +```py +reveal_type(TypeVarAndParamSpec[int, Any]().attr) # revealed: (...) -> int +``` + +## Specialization when defaults are involved + +```py +from typing import Callable, ParamSpec + +class ParamSpecWithDefault1[**P1 = [int, str]]: + attr: Callable[P1, None] + +reveal_type(ParamSpecWithDefault1().attr) # revealed: (int, str, /) -> None +reveal_type(ParamSpecWithDefault1[int]().attr) # revealed: (int, /) -> None +``` + +```py +class ParamSpecWithDefault2[**P1 = ...]: + attr: Callable[P1, None] + +reveal_type(ParamSpecWithDefault2().attr) # revealed: (...) -> None +reveal_type(ParamSpecWithDefault2[int, str]().attr) # revealed: (int, str, /) -> None +``` + +```py +class ParamSpecWithDefault3[**P1, **P2 = P1]: + attr1: Callable[P1, None] + attr2: Callable[P2, None] + +# `P1` hasn't been specialized, so it defaults to `...` gradual form +p1 = ParamSpecWithDefault3() +reveal_type(p1.attr1) # revealed: (...) -> None +reveal_type(p1.attr2) # revealed: (...) -> None + +p2 = ParamSpecWithDefault3[[int, str]]() +reveal_type(p2.attr1) # revealed: (int, str, /) -> None +reveal_type(p2.attr2) # revealed: (int, str, /) -> None + +p3 = ParamSpecWithDefault3[[int], [str]]() +reveal_type(p3.attr1) # revealed: (int, /) -> None +reveal_type(p3.attr2) # revealed: (str, /) -> None + +class ParamSpecWithDefault4[**P1 = [int, str], **P2 = P1]: + attr1: Callable[P1, None] + attr2: Callable[P2, None] + +p1 = ParamSpecWithDefault4() +reveal_type(p1.attr1) # revealed: (int, str, /) -> None +reveal_type(p1.attr2) # revealed: (int, str, /) -> None + +p2 = ParamSpecWithDefault4[[int]]() +reveal_type(p2.attr1) # revealed: (int, /) -> None +reveal_type(p2.attr2) # revealed: (int, /) -> None + +p3 = ParamSpecWithDefault4[[int], [str]]() +reveal_type(p3.attr1) # revealed: (int, /) -> None +reveal_type(p3.attr2) # revealed: (str, /) -> None + +P2 = ParamSpec("P2") + +# TODO: error: paramspec is out of scope +class ParamSpecWithDefault5[**P1 = P2]: + attr: Callable[P1, None] +``` + +## Semantics + +Most of these test cases are adopted from the +[typing documentation on `ParamSpec` semantics](https://typing.python.org/en/latest/spec/generics.html#semantics). + +### Return type change using `ParamSpec` once + +```py +from typing import Callable + +def converter[**P](func: Callable[P, int]) -> Callable[P, bool]: + def wrapper(*args: P.args, **kwargs: P.kwargs) -> bool: + func(*args, **kwargs) + return True + return wrapper + +def f1(x: int, y: str) -> int: + return 1 + +# This should preserve all the information about the parameters of `f1` +f2 = converter(f1) + +reveal_type(f2) # revealed: (x: int, y: str) -> bool + +reveal_type(f1(1, "a")) # revealed: int +reveal_type(f2(1, "a")) # revealed: bool + +# As it preserves the parameter kinds, the following should work as well +reveal_type(f2(1, y="a")) # revealed: bool +reveal_type(f2(x=1, y="a")) # revealed: bool +reveal_type(f2(y="a", x=1)) # revealed: bool + +# error: [missing-argument] "No argument provided for required parameter `y`" +f2(1) +# error: [invalid-argument-type] "Argument is incorrect: Expected `int`, found `Literal["a"]`" +f2("a", "b") +``` + +The `converter` function act as a decorator here: + +```py +@converter +def f3(x: int, y: str) -> int: + return 1 + +# TODO: This should reveal `(x: int, y: str) -> bool` but there's a cycle: https://github.com/astral-sh/ty/issues/1729 +reveal_type(f3) # revealed: ((x: int, y: str) -> bool) | ((x: Divergent, y: Divergent) -> bool) + +reveal_type(f3(1, "a")) # revealed: bool +reveal_type(f3(x=1, y="a")) # revealed: bool +reveal_type(f3(1, y="a")) # revealed: bool +reveal_type(f3(y="a", x=1)) # revealed: bool + +# TODO: There should only be one error but the type of `f3` is a union: https://github.com/astral-sh/ty/issues/1729 +# error: [missing-argument] "No argument provided for required parameter `y`" +# error: [missing-argument] "No argument provided for required parameter `y`" +f3(1) +# error: [invalid-argument-type] "Argument is incorrect: Expected `int`, found `Literal["a"]`" +f3("a", "b") +``` + +### Return type change using the same `ParamSpec` multiple times + +```py +from typing import Callable + +def multiple[**P](func1: Callable[P, int], func2: Callable[P, int]) -> Callable[P, bool]: + def wrapper(*args: P.args, **kwargs: P.kwargs) -> bool: + func1(*args, **kwargs) + func2(*args, **kwargs) + return True + return wrapper +``` + +As per the spec, + +> A user may include the same `ParamSpec` multiple times in the arguments of the same function, to +> indicate a dependency between multiple arguments. In these cases a type checker may choose to +> solve to a common behavioral supertype (i.e. a set of parameters for which all of the valid calls +> are valid in both of the subtypes), but is not obligated to do so. + +TODO: Currently, we don't do this + +```py +def xy(x: int, y: str) -> int: + return 1 + +def yx(y: int, x: str) -> int: + return 2 + +reveal_type(multiple(xy, xy)) # revealed: (x: int, y: str) -> bool + +# The common supertype is `(int, str, /)` which is converting the positional-or-keyword parameters +# into positional-only parameters because the position of the types are the same. +# TODO: This shouldn't error +# error: [invalid-argument-type] +reveal_type(multiple(xy, yx)) # revealed: (x: int, y: str) -> bool + +def keyword_only_with_default_1(*, x: int = 42) -> int: + return 1 + +def keyword_only_with_default_2(*, y: int = 42) -> int: + return 2 + +# The common supertype for two functions with only keyword-only parameters would be an empty +# parameter list i.e., `()` +# TODO: This shouldn't error +# error: [invalid-argument-type] +# revealed: (*, x: int = Literal[42]) -> bool +reveal_type(multiple(keyword_only_with_default_1, keyword_only_with_default_2)) + +def keyword_only1(*, x: int) -> int: + return 1 + +def keyword_only2(*, y: int) -> int: + return 2 + +# On the other hand, combining two functions with only keyword-only parameters does not have a +# common supertype, so it should result in an error. +# error: [invalid-argument-type] "Argument to function `multiple` is incorrect: Expected `(*, x: int) -> int`, found `def keyword_only2(*, y: int) -> int`" +reveal_type(multiple(keyword_only1, keyword_only2)) # revealed: (*, x: int) -> bool +``` + +### Constructors of user-defined generic class on `ParamSpec` + +```py +from typing import Callable + +class C[**P]: + f: Callable[P, int] + + def __init__(self, f: Callable[P, int]) -> None: + self.f = f + +def f(x: int, y: str) -> bool: + return True + +c = C(f) +reveal_type(c.f) # revealed: (x: int, y: str) -> int +``` + +### `ParamSpec` in prepended positional parameters + +> If one of these prepended positional parameters contains a free `ParamSpec`, we consider that +> variable in scope for the purposes of extracting the components of that `ParamSpec`. + +```py +from typing import Callable + +def foo1[**P1](func: Callable[P1, int], *args: P1.args, **kwargs: P1.kwargs) -> int: + return func(*args, **kwargs) + +def foo1_with_extra_arg[**P1](func: Callable[P1, int], extra: str, *args: P1.args, **kwargs: P1.kwargs) -> int: + return func(*args, **kwargs) + +def foo2[**P2](func: Callable[P2, int], *args: P2.args, **kwargs: P2.kwargs) -> None: + foo1(func, *args, **kwargs) + + # error: [invalid-argument-type] "Argument to function `foo1` is incorrect: Expected `P2@foo2.args`, found `Literal[1]`" + foo1(func, 1, *args, **kwargs) + + # error: [invalid-argument-type] "Argument to function `foo1_with_extra_arg` is incorrect: Expected `str`, found `P2@foo2.args`" + foo1_with_extra_arg(func, *args, **kwargs) + + foo1_with_extra_arg(func, "extra", *args, **kwargs) +``` + +Here, the first argument to `f` can specialize `P` to the parameters of the callable passed to it +which is then used to type the `ParamSpec` components used in `*args` and `**kwargs`. + +```py +def f1(x: int, y: str) -> int: + return 1 + +foo1(f1, 1, "a") +foo1(f1, x=1, y="a") +foo1(f1, 1, y="a") + +# error: [missing-argument] "No arguments provided for required parameters `x`, `y` of function `foo1`" +foo1(f1) + +# error: [missing-argument] "No argument provided for required parameter `y` of function `foo1`" +foo1(f1, 1) + +# error: [invalid-argument-type] "Argument to function `foo1` is incorrect: Expected `str`, found `Literal[2]`" +foo1(f1, 1, 2) + +# error: [too-many-positional-arguments] "Too many positional arguments to function `foo1`: expected 2, got 3" +foo1(f1, 1, "a", "b") + +# error: [missing-argument] "No argument provided for required parameter `y` of function `foo1`" +# error: [unknown-argument] "Argument `z` does not match any known parameter of function `foo1`" +foo1(f1, x=1, z="a") +``` + +### Specializing `ParamSpec` with another `ParamSpec` + +```py +class Foo[**P]: + def __init__(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.args = args + self.kwargs = kwargs + +def bar[**P](foo: Foo[P]) -> None: + reveal_type(foo) # revealed: Foo[P@bar] + reveal_type(foo.args) # revealed: Unknown | P@bar.args + reveal_type(foo.kwargs) # revealed: Unknown | P@bar.kwargs +``` + +ty will check whether the argument after `**` is a mapping type but as instance attribute are +unioned with `Unknown`, it shouldn't error here. + +```py +from typing import Callable + +def baz[**P](fn: Callable[P, None], foo: Foo[P]) -> None: + fn(*foo.args, **foo.kwargs) +``` + +The `Unknown` can be eliminated by using annotating these attributes with `Final`: + +```py +from typing import Final + +class FooWithFinal[**P]: + def __init__(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.args: Final = args + self.kwargs: Final = kwargs + +def with_final[**P](foo: FooWithFinal[P]) -> None: + reveal_type(foo) # revealed: FooWithFinal[P@with_final] + reveal_type(foo.args) # revealed: P@with_final.args + reveal_type(foo.kwargs) # revealed: P@with_final.kwargs +``` + +### Specializing `Self` when `ParamSpec` is involved + +```py +class Foo[**P]: + def method(self, *args: P.args, **kwargs: P.kwargs) -> str: + return "hello" + +foo = Foo[int, str]() + +reveal_type(foo) # revealed: Foo[(int, str, /)] +reveal_type(foo.method) # revealed: bound method Foo[(int, str, /)].method(int, str, /) -> str +reveal_type(foo.method(1, "a")) # revealed: str +``` + +### Overloads + +`overloaded.pyi`: + +```pyi +from typing import overload + +@overload +def int_int(x: int) -> int: ... +@overload +def int_int(x: str) -> int: ... + +@overload +def int_str(x: int) -> int: ... +@overload +def int_str(x: str) -> str: ... + +@overload +def str_str(x: int) -> str: ... +@overload +def str_str(x: str) -> str: ... +``` + +```py +from typing import Callable +from overloaded import int_int, int_str, str_str + +def change_return_type[**P](f: Callable[P, int]) -> Callable[P, str]: + def nested(*args: P.args, **kwargs: P.kwargs) -> str: + return str(f(*args, **kwargs)) + return nested + +def with_parameters[**P](f: Callable[P, int], *args: P.args, **kwargs: P.kwargs) -> Callable[P, str]: + def nested(*args: P.args, **kwargs: P.kwargs) -> str: + return str(f(*args, **kwargs)) + return nested + +reveal_type(change_return_type(int_int)) # revealed: Overload[(x: int) -> str, (x: str) -> str] + +# TODO: This shouldn't error and should pick the first overload because of the return type +# error: [invalid-argument-type] +reveal_type(change_return_type(int_str)) # revealed: Overload[(x: int) -> str, (x: str) -> str] + +# error: [invalid-argument-type] +reveal_type(change_return_type(str_str)) # revealed: Overload[(x: int) -> str, (x: str) -> str] + +# TODO: Both of these shouldn't raise an error +# error: [invalid-argument-type] +reveal_type(with_parameters(int_int, 1)) # revealed: Overload[(x: int) -> str, (x: str) -> str] +# error: [invalid-argument-type] +reveal_type(with_parameters(int_int, "a")) # revealed: Overload[(x: int) -> str, (x: str) -> str] +``` diff --git a/crates/ty_python_semantic/resources/mdtest/implicit_type_aliases.md b/crates/ty_python_semantic/resources/mdtest/implicit_type_aliases.md index d0836559f3..0886393143 100644 --- a/crates/ty_python_semantic/resources/mdtest/implicit_type_aliases.md +++ b/crates/ty_python_semantic/resources/mdtest/implicit_type_aliases.md @@ -398,7 +398,7 @@ reveal_type(Sum) # revealed: reveal_type(ListOrTuple) # revealed: # revealed: reveal_type(ListOrTupleLegacy) -reveal_type(MyCallable) # revealed: @Todo(Callable[..] specialized with ParamSpec) +reveal_type(MyCallable) # revealed: T@MyCallable'> reveal_type(AnnotatedType) # revealed: ]'> reveal_type(TransparentAlias) # revealed: typing.TypeVar reveal_type(MyOptional) # revealed: @@ -425,8 +425,7 @@ def _( reveal_type(int_and_bytes) # revealed: tuple[int, bytes] reveal_type(list_or_tuple) # revealed: list[int] | tuple[int, ...] reveal_type(list_or_tuple_legacy) # revealed: list[int] | tuple[int, ...] - # TODO: This should be `(str, bytes) -> int` - reveal_type(my_callable) # revealed: @Todo(Callable[..] specialized with ParamSpec) + reveal_type(my_callable) # revealed: (str, bytes, /) -> int reveal_type(annotated_int) # revealed: int reveal_type(transparent_alias) # revealed: int reveal_type(optional_int) # revealed: int | None @@ -463,7 +462,7 @@ reveal_type(ListOfPairs) # revealed: reveal_type(ListOrTupleOfInts) # revealed: reveal_type(AnnotatedInt) # revealed: ]'> reveal_type(SubclassOfInt) # revealed: -reveal_type(CallableIntToStr) # revealed: @Todo(Callable[..] specialized with ParamSpec) +reveal_type(CallableIntToStr) # revealed: str'> def _( ints_or_none: IntsOrNone, @@ -480,8 +479,7 @@ def _( reveal_type(list_or_tuple_of_ints) # revealed: list[int] | tuple[int, ...] reveal_type(annotated_int) # revealed: int reveal_type(subclass_of_int) # revealed: type[int] - # TODO: This should be `(int, /) -> str` - reveal_type(callable_int_to_str) # revealed: @Todo(Callable[..] specialized with ParamSpec) + reveal_type(callable_int_to_str) # revealed: (int, /) -> str ``` A generic implicit type alias can also be used in another generic implicit type alias: @@ -534,8 +532,7 @@ def _( reveal_type(unknown_and_unknown) # revealed: tuple[Unknown, Unknown] reveal_type(list_or_tuple) # revealed: list[Unknown] | tuple[Unknown, ...] reveal_type(list_or_tuple_legacy) # revealed: list[Unknown] | tuple[Unknown, ...] - # TODO: should be (...) -> Unknown - reveal_type(my_callable) # revealed: @Todo(Callable[..] specialized with ParamSpec) + reveal_type(my_callable) # revealed: (...) -> Unknown reveal_type(annotated_unknown) # revealed: Unknown reveal_type(optional_unknown) # revealed: Unknown | None ``` diff --git a/crates/ty_python_semantic/resources/mdtest/type_properties/is_assignable_to.md b/crates/ty_python_semantic/resources/mdtest/type_properties/is_assignable_to.md index 885d3c2e4f..3bd3cd1f09 100644 --- a/crates/ty_python_semantic/resources/mdtest/type_properties/is_assignable_to.md +++ b/crates/ty_python_semantic/resources/mdtest/type_properties/is_assignable_to.md @@ -1344,6 +1344,38 @@ static_assert(not is_assignable_to(TypeGuard[Unknown], str)) # error: [static-a static_assert(not is_assignable_to(TypeIs[Any], str)) ``` +## `ParamSpec` + +```py +from ty_extensions import TypeOf, static_assert, is_assignable_to, Unknown +from typing import ParamSpec, Mapping, Callable, Any + +P = ParamSpec("P") + +def f(func: Callable[P, int], *args: P.args, **kwargs: P.kwargs) -> None: + static_assert(is_assignable_to(TypeOf[args], tuple[Any, ...])) + static_assert(is_assignable_to(TypeOf[args], tuple[object, ...])) + static_assert(is_assignable_to(TypeOf[args], tuple[Unknown, ...])) + static_assert(not is_assignable_to(TypeOf[args], tuple[int, ...])) + static_assert(not is_assignable_to(TypeOf[args], tuple[int, str])) + + static_assert(not is_assignable_to(tuple[Any, ...], TypeOf[args])) + static_assert(not is_assignable_to(tuple[object, ...], TypeOf[args])) + static_assert(not is_assignable_to(tuple[Unknown, ...], TypeOf[args])) + + static_assert(is_assignable_to(TypeOf[kwargs], dict[str, Any])) + static_assert(is_assignable_to(TypeOf[kwargs], dict[str, Unknown])) + static_assert(not is_assignable_to(TypeOf[kwargs], dict[str, object])) + static_assert(not is_assignable_to(TypeOf[kwargs], dict[str, int])) + static_assert(is_assignable_to(TypeOf[kwargs], Mapping[str, Any])) + static_assert(is_assignable_to(TypeOf[kwargs], Mapping[str, object])) + static_assert(is_assignable_to(TypeOf[kwargs], Mapping[str, Unknown])) + + static_assert(not is_assignable_to(dict[str, Any], TypeOf[kwargs])) + static_assert(not is_assignable_to(dict[str, object], TypeOf[kwargs])) + static_assert(not is_assignable_to(dict[str, Unknown], TypeOf[kwargs])) +``` + [gradual form]: https://typing.python.org/en/latest/spec/glossary.html#term-gradual-form [gradual tuple]: https://typing.python.org/en/latest/spec/tuples.html#tuple-type-form [typing documentation]: https://typing.python.org/en/latest/spec/concepts.html#the-assignable-to-or-consistent-subtyping-relation diff --git a/crates/ty_python_semantic/resources/mdtest/with/async.md b/crates/ty_python_semantic/resources/mdtest/with/async.md index 6d556d438b..2a0d7165de 100644 --- a/crates/ty_python_semantic/resources/mdtest/with/async.md +++ b/crates/ty_python_semantic/resources/mdtest/with/async.md @@ -213,7 +213,7 @@ async def connect() -> AsyncGenerator[Session]: yield Session() # TODO: this should be `() -> _AsyncGeneratorContextManager[Session, None]` -reveal_type(connect) # revealed: (...) -> _AsyncGeneratorContextManager[Unknown, None] +reveal_type(connect) # revealed: () -> _AsyncGeneratorContextManager[Unknown, None] async def main(): async with connect() as session: diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 6022aad53e..23f7a53f79 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -1796,14 +1796,14 @@ impl<'db> Type<'db> { Type::KnownBoundMethod(method) => Some(CallableTypes::one(CallableType::new( db, CallableSignature::from_overloads(method.signatures(db)), - false, + CallableTypeKind::Regular, ))), Type::WrapperDescriptor(wrapper_descriptor) => { Some(CallableTypes::one(CallableType::new( db, CallableSignature::from_overloads(wrapper_descriptor.signatures(db)), - false, + CallableTypeKind::Regular, ))) } @@ -4986,11 +4986,17 @@ impl<'db> Type<'db> { .into() } - Type::KnownInstance(KnownInstanceType::TypeVar(typevar)) - if typevar.kind(db).is_paramspec() - && matches!(name.as_str(), "args" | "kwargs") => - { - Place::bound(todo_type!("ParamSpecArgs / ParamSpecKwargs")).into() + Type::TypeVar(typevar) if name_str == "args" && typevar.is_paramspec(db) => { + Place::declared(Type::TypeVar( + typevar.with_paramspec_attr(db, ParamSpecAttrKind::Args), + )) + .into() + } + Type::TypeVar(typevar) if name_str == "kwargs" && typevar.is_paramspec(db) => { + Place::declared(Type::TypeVar( + typevar.with_paramspec_attr(db, ParamSpecAttrKind::Kwargs), + )) + .into() } Type::NominalInstance(instance) @@ -7197,6 +7203,9 @@ impl<'db> Type<'db> { KnownInstanceType::TypeAliasType(alias) => Ok(Type::TypeAlias(*alias)), KnownInstanceType::NewType(newtype) => Ok(Type::NewTypeInstance(*newtype)), KnownInstanceType::TypeVar(typevar) => { + // TODO: A `ParamSpec` type variable cannot be used in type expressions. This + // requires storing additional context as it's allowed in some places + // (`Concatenate`, `Callable`) but not others. let index = semantic_index(db, scope_id.file(db)); Ok(bind_typevar( db, @@ -7423,9 +7432,6 @@ impl<'db> Type<'db> { Some(KnownClass::TypeVar) => Ok(todo_type!( "Support for `typing.TypeVar` instances in type expressions" )), - Some( - KnownClass::ParamSpec | KnownClass::ParamSpecArgs | KnownClass::ParamSpecKwargs, - ) => Ok(todo_type!("Support for `typing.ParamSpec`")), Some(KnownClass::TypeVarTuple) => Ok(todo_type!( "Support for `typing.TypeVarTuple` instances in type expressions" )), @@ -7604,7 +7610,7 @@ impl<'db> Type<'db> { KnownInstanceType::TypeVar(typevar) => { match type_mapping { TypeMapping::BindLegacyTypevars(binding_context) => { - Type::TypeVar(BoundTypeVarInstance::new(db, typevar, *binding_context)) + Type::TypeVar(BoundTypeVarInstance::new(db, typevar, *binding_context, None)) } TypeMapping::Specialization(_) | TypeMapping::PartialSpecialization(_) | @@ -7858,18 +7864,28 @@ impl<'db> Type<'db> { typevars: &mut FxOrderSet>, visitor: &FindLegacyTypeVarsVisitor<'db>, ) { - let is_matching_typevar = |bound_typevar: &BoundTypeVarInstance<'db>| { - matches!( - bound_typevar.typevar(db).kind(db), - TypeVarKind::Legacy | TypeVarKind::TypingSelf | TypeVarKind::ParamSpec - ) && binding_context.is_none_or(|binding_context| { - bound_typevar.binding_context(db) == BindingContext::Definition(binding_context) - }) + let matching_typevar = |bound_typevar: &BoundTypeVarInstance<'db>| { + match bound_typevar.typevar(db).kind(db) { + TypeVarKind::Legacy | TypeVarKind::TypingSelf + if binding_context.is_none_or(|binding_context| { + bound_typevar.binding_context(db) + == BindingContext::Definition(binding_context) + }) => + { + Some(*bound_typevar) + } + TypeVarKind::ParamSpec => { + // For `ParamSpec`, we're only interested in `P` itself, not `P.args` or + // `P.kwargs`. + Some(bound_typevar.without_paramspec_attr(db)) + } + _ => None, + } }; match self { Type::TypeVar(bound_typevar) => { - if is_matching_typevar(&bound_typevar) { + if let Some(bound_typevar) = matching_typevar(&bound_typevar) { typevars.insert(bound_typevar); } } @@ -8011,7 +8027,7 @@ impl<'db> Type<'db> { Type::Dynamic(DynamicType::UnknownGeneric(generic_context)) => { for variable in generic_context.variables(db) { - if is_matching_typevar(&variable) { + if let Some(variable) = matching_typevar(&variable) { typevars.insert(variable); } } @@ -8813,7 +8829,7 @@ impl<'db> KnownInstanceType<'db> { fn class(self, db: &'db dyn Db) -> KnownClass { match self { Self::SubscriptedProtocol(_) | Self::SubscriptedGeneric(_) => KnownClass::SpecialForm, - Self::TypeVar(typevar_instance) if typevar_instance.kind(db).is_paramspec() => { + Self::TypeVar(typevar_instance) if typevar_instance.is_paramspec(db) => { KnownClass::ParamSpec } Self::TypeVar(_) => KnownClass::TypeVar, @@ -9461,7 +9477,7 @@ impl<'db> TypeVarInstance<'db> { db: &'db dyn Db, binding_context: Definition<'db>, ) -> BoundTypeVarInstance<'db> { - BoundTypeVarInstance::new(db, self, BindingContext::Definition(binding_context)) + BoundTypeVarInstance::new(db, self, BindingContext::Definition(binding_context), None) } pub(crate) fn name(self, db: &'db dyn Db) -> &'db ast::name::Name { @@ -9480,6 +9496,10 @@ impl<'db> TypeVarInstance<'db> { matches!(self.kind(db), TypeVarKind::TypingSelf) } + pub(crate) fn is_paramspec(self, db: &'db dyn Db) -> bool { + self.kind(db).is_paramspec() + } + pub(crate) fn upper_bound(self, db: &'db dyn Db) -> Option> { if let Some(TypeVarBoundOrConstraints::UpperBound(ty)) = self.bound_or_constraints(db) { Some(ty) @@ -9683,6 +9703,45 @@ impl<'db> TypeVarInstance<'db> { #[salsa::tracked(cycle_fn=lazy_default_cycle_recover, cycle_initial=lazy_default_cycle_initial, heap_size=ruff_memory_usage::heap_size)] fn lazy_default(self, db: &'db dyn Db) -> Option> { + fn convert_type_to_paramspec_value<'db>(db: &'db dyn Db, ty: Type<'db>) -> Type<'db> { + let parameters = match ty { + Type::NominalInstance(nominal_instance) + if nominal_instance.has_known_class(db, KnownClass::EllipsisType) => + { + Parameters::gradual_form() + } + Type::NominalInstance(nominal_instance) => nominal_instance + .own_tuple_spec(db) + .map_or_else(Parameters::unknown, |tuple_spec| { + Parameters::new( + db, + tuple_spec.all_elements().map(|ty| { + Parameter::positional_only(None).with_annotated_type(*ty) + }), + ) + }), + Type::Dynamic(dynamic) => match dynamic { + DynamicType::Todo(_) + | DynamicType::TodoUnpack + | DynamicType::TodoStarredExpression => Parameters::todo(), + DynamicType::Any + | DynamicType::Unknown + | DynamicType::UnknownGeneric(_) + | DynamicType::Divergent(_) => Parameters::unknown(), + }, + Type::TypeVar(typevar) if typevar.is_paramspec(db) => { + return ty; + } + Type::KnownInstance(KnownInstanceType::TypeVar(typevar)) + if typevar.is_paramspec(db) => + { + return ty; + } + _ => Parameters::unknown(), + }; + Type::paramspec_value_callable(db, parameters) + } + let definition = self.definition(db)?; let module = parsed_module(db, definition.file(db)).load(db); match definition.kind(db) { @@ -9695,27 +9754,35 @@ impl<'db> TypeVarInstance<'db> { typevar_node.default.as_ref()?, )) } - // legacy typevar + // legacy typevar / ParamSpec DefinitionKind::Assignment(assignment) => { let call_expr = assignment.value(&module).as_call_expr()?; + let func_ty = definition_expression_type(db, definition, &call_expr.func); + let known_class = func_ty.as_class_literal().and_then(|cls| cls.known(db)); let expr = &call_expr.arguments.find_keyword("default")?.value; - Some(definition_expression_type(db, definition, expr)) + let default_type = definition_expression_type(db, definition, expr); + if known_class == Some(KnownClass::ParamSpec) { + Some(convert_type_to_paramspec_value(db, default_type)) + } else { + Some(default_type) + } } // PEP 695 ParamSpec DefinitionKind::ParamSpec(paramspec) => { let paramspec_node = paramspec.node(&module); - Some(definition_expression_type( - db, - definition, - paramspec_node.default.as_ref()?, - )) + let default_ty = + definition_expression_type(db, definition, paramspec_node.default.as_ref()?); + Some(convert_type_to_paramspec_value(db, default_ty)) } _ => None, } } pub fn bind_pep695(self, db: &'db dyn Db) -> Option> { - if self.identity(db).kind(db) != TypeVarKind::Pep695 { + if !matches!( + self.identity(db).kind(db), + TypeVarKind::Pep695 | TypeVarKind::Pep695ParamSpec + ) { return None; } let typevar_definition = self.definition(db)?; @@ -9810,6 +9877,21 @@ impl<'db> BindingContext<'db> { } } +#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, get_size2::GetSize)] +pub enum ParamSpecAttrKind { + Args, + Kwargs, +} + +impl std::fmt::Display for ParamSpecAttrKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ParamSpecAttrKind::Args => f.write_str("args"), + ParamSpecAttrKind::Kwargs => f.write_str("kwargs"), + } + } +} + /// The identity of a bound type variable. /// /// This identifies a specific binding of a typevar to a context (e.g., `T@ClassC` vs `T@FunctionF`), @@ -9822,14 +9904,26 @@ impl<'db> BindingContext<'db> { pub struct BoundTypeVarIdentity<'db> { pub(crate) identity: TypeVarIdentity<'db>, pub(crate) binding_context: BindingContext<'db>, + /// If [`Some`], this indicates that this type variable is the `args` or `kwargs` component + /// of a `ParamSpec` i.e., `P.args` or `P.kwargs`. + paramspec_attr: Option, } /// A type variable that has been bound to a generic context, and which can be specialized to a /// concrete type. +/// +/// # Ordering +/// +/// Ordering is based on the wrapped data's salsa-assigned id and not on its values. +/// The id may change between runs, or when e.g. a `BoundTypeVarInstance` was garbage-collected and recreated. #[salsa::interned(debug, heap_size=ruff_memory_usage::heap_size)] +#[derive(PartialOrd, Ord)] pub struct BoundTypeVarInstance<'db> { pub typevar: TypeVarInstance<'db>, binding_context: BindingContext<'db>, + /// If [`Some`], this indicates that this type variable is the `args` or `kwargs` component + /// of a `ParamSpec` i.e., `P.args` or `P.kwargs`. + paramspec_attr: Option, } // The Salsa heap is tracked separately. @@ -9844,9 +9938,83 @@ impl<'db> BoundTypeVarInstance<'db> { BoundTypeVarIdentity { identity: self.typevar(db).identity(db), binding_context: self.binding_context(db), + paramspec_attr: self.paramspec_attr(db), } } + pub(crate) fn name(self, db: &'db dyn Db) -> &'db ast::name::Name { + self.typevar(db).name(db) + } + + pub(crate) fn kind(self, db: &'db dyn Db) -> TypeVarKind { + self.typevar(db).kind(db) + } + + pub(crate) fn is_paramspec(self, db: &'db dyn Db) -> bool { + self.kind(db).is_paramspec() + } + + /// Returns a new bound typevar instance with the given `ParamSpec` attribute set. + /// + /// This method will also set an appropriate upper bound on the typevar, based on the + /// attribute kind. For `P.args`, the upper bound will be `tuple[object, ...]`, and for + /// `P.kwargs`, the upper bound will be `Top[dict[str, Any]]`. + /// + /// It's the caller's responsibility to ensure that this method is only called on a `ParamSpec` + /// type variable. + pub(crate) fn with_paramspec_attr(self, db: &'db dyn Db, kind: ParamSpecAttrKind) -> Self { + debug_assert!( + self.is_paramspec(db), + "Expected a ParamSpec, got {:?}", + self.kind(db) + ); + + let upper_bound = TypeVarBoundOrConstraints::UpperBound(match kind { + ParamSpecAttrKind::Args => Type::homogeneous_tuple(db, Type::object()), + ParamSpecAttrKind::Kwargs => KnownClass::Dict + .to_specialized_instance(db, [KnownClass::Str.to_instance(db), Type::any()]) + .top_materialization(db), + }); + + let typevar = TypeVarInstance::new( + db, + self.typevar(db).identity(db), + Some(TypeVarBoundOrConstraintsEvaluation::Eager(upper_bound)), + None, // ParamSpecs cannot have explicit variance + None, // `P.args` and `P.kwargs` cannot have defaults even though `P` can + ); + + Self::new(db, typevar, self.binding_context(db), Some(kind)) + } + + /// Returns a new bound typevar instance without any `ParamSpec` attribute set. + /// + /// This method will also remove any upper bound that was set by `with_paramspec_attr`. This + /// means that the returned typevar will have no upper bound or constraints. + /// + /// It's the caller's responsibility to ensure that this method is only called on a `ParamSpec` + /// type variable. + pub(crate) fn without_paramspec_attr(self, db: &'db dyn Db) -> Self { + debug_assert!( + self.is_paramspec(db), + "Expected a ParamSpec, got {:?}", + self.kind(db) + ); + + Self::new( + db, + TypeVarInstance::new( + db, + self.typevar(db).identity(db), + None, // Remove the upper bound set by `with_paramspec_attr` + None, // ParamSpecs cannot have explicit variance + None, // `P.args` and `P.kwargs` cannot have defaults even though `P` can + ), + self.binding_context(db), + None, + ) + } + /// Returns whether two bound typevars represent the same logical typevar, regardless of e.g. /// differences in their bounds or constraints due to materialization. pub(crate) fn is_same_typevar_as(self, db: &'db dyn Db, other: Self) -> bool { @@ -9873,7 +10041,7 @@ impl<'db> BoundTypeVarInstance<'db> { Some(variance), None, // _default ); - Self::new(db, typevar, BindingContext::Synthetic) + Self::new(db, typevar, BindingContext::Synthetic, None) } /// Create a new synthetic `Self` type variable with the given upper bound. @@ -9895,7 +10063,7 @@ impl<'db> BoundTypeVarInstance<'db> { Some(TypeVarVariance::Invariant), None, // _default ); - Self::new(db, typevar, binding_context) + Self::new(db, typevar, binding_context, None) } /// Returns an identical type variable with its `TypeVarBoundOrConstraints` mapped by the @@ -9914,7 +10082,12 @@ impl<'db> BoundTypeVarInstance<'db> { self.typevar(db)._default(db), ); - Self::new(db, typevar, self.binding_context(db)) + Self::new( + db, + typevar, + self.binding_context(db), + self.paramspec_attr(db), + ) } pub(crate) fn variance_with_polarity( @@ -9946,10 +10119,42 @@ impl<'db> BoundTypeVarInstance<'db> { ) -> Type<'db> { match type_mapping { TypeMapping::Specialization(specialization) => { - specialization.get(db, self).unwrap_or(Type::TypeVar(self)) + let typevar = if self.is_paramspec(db) { + self.without_paramspec_attr(db) + } else { + self + }; + specialization + .get(db, typevar) + .map(|ty| { + if let Some(attr) = self.paramspec_attr(db) + && let Type::TypeVar(typevar) = ty + && typevar.is_paramspec(db) + { + return Type::TypeVar(typevar.with_paramspec_attr(db, attr)); + } + ty + }) + .unwrap_or(Type::TypeVar(self)) } TypeMapping::PartialSpecialization(partial) => { - partial.get(db, self).unwrap_or(Type::TypeVar(self)) + let typevar = if self.is_paramspec(db) { + self.without_paramspec_attr(db) + } else { + self + }; + partial + .get(db, typevar) + .map(|ty| { + if let Some(attr) = self.paramspec_attr(db) + && let Type::TypeVar(typevar) = ty + && typevar.is_paramspec(db) + { + return Type::TypeVar(typevar.with_paramspec_attr(db, attr)); + } + ty + }) + .unwrap_or(Type::TypeVar(self)) } TypeMapping::BindSelf { self_type, @@ -10032,6 +10237,7 @@ impl<'db> BoundTypeVarInstance<'db> { db, self.typevar(db).normalized_impl(db, visitor), self.binding_context(db), + self.paramspec_attr(db), ) } @@ -10046,6 +10252,7 @@ impl<'db> BoundTypeVarInstance<'db> { self.typevar(db) .materialize_impl(db, materialization_kind, visitor), self.binding_context(db), + self.paramspec_attr(db), ) } @@ -10054,6 +10261,7 @@ impl<'db> BoundTypeVarInstance<'db> { db, self.typevar(db).to_instance(db)?, self.binding_context(db), + self.paramspec_attr(db), )) } } @@ -11797,7 +12005,7 @@ impl<'db> BoundMethodType<'db> { .iter() .map(|signature| signature.bind_self(db, Some(self_instance))), ), - true, + CallableTypeKind::FunctionLike, ) } @@ -11878,6 +12086,20 @@ impl<'db> BoundMethodType<'db> { } } +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, get_size2::GetSize)] +pub enum CallableTypeKind { + /// Represents regular callable objects. + Regular, + + /// Represents function-like objects, like the synthesized methods of dataclasses or + /// `NamedTuples`. These callables act like real functions when accessed as attributes on + /// instances, i.e. they bind `self`. + FunctionLike, + + /// Represents the value bound to a `typing.ParamSpec` type variable. + ParamSpecValue, +} + /// This type represents the set of all callable objects with a certain, possibly overloaded, /// signature. /// @@ -11894,10 +12116,7 @@ pub struct CallableType<'db> { #[returns(ref)] pub(crate) signatures: CallableSignature<'db>, - /// We use `CallableType` to represent function-like objects, like the synthesized methods - /// of dataclasses or NamedTuples. These callables act like real functions when accessed - /// as attributes on instances, i.e. they bind `self`. - is_function_like: bool, + kind: CallableTypeKind, } pub(super) fn walk_callable_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>( @@ -11925,15 +12144,43 @@ impl<'db> Type<'db> { pub(crate) fn function_like_callable(db: &'db dyn Db, signature: Signature<'db>) -> Type<'db> { Type::Callable(CallableType::function_like(db, signature)) } + + /// Create a non-overloaded callable type which represents the value bound to a `ParamSpec` + /// type variable. + pub(crate) fn paramspec_value_callable( + db: &'db dyn Db, + parameters: Parameters<'db>, + ) -> Type<'db> { + Type::Callable(CallableType::paramspec_value(db, parameters)) + } } impl<'db> CallableType<'db> { pub(crate) fn single(db: &'db dyn Db, signature: Signature<'db>) -> CallableType<'db> { - CallableType::new(db, CallableSignature::single(signature), false) + CallableType::new( + db, + CallableSignature::single(signature), + CallableTypeKind::Regular, + ) } pub(crate) fn function_like(db: &'db dyn Db, signature: Signature<'db>) -> CallableType<'db> { - CallableType::new(db, CallableSignature::single(signature), true) + CallableType::new( + db, + CallableSignature::single(signature), + CallableTypeKind::FunctionLike, + ) + } + + pub(crate) fn paramspec_value( + db: &'db dyn Db, + parameters: Parameters<'db>, + ) -> CallableType<'db> { + CallableType::new( + db, + CallableSignature::single(Signature::new(parameters, None)), + CallableTypeKind::ParamSpecValue, + ) } /// Create a callable type which accepts any parameters and returns an `Unknown` type. @@ -11941,6 +12188,10 @@ impl<'db> CallableType<'db> { Self::single(db, Signature::unknown()) } + pub(crate) fn is_function_like(self, db: &'db dyn Db) -> bool { + matches!(self.kind(db), CallableTypeKind::FunctionLike) + } + pub(crate) fn bind_self( self, db: &'db dyn Db, @@ -11949,7 +12200,7 @@ impl<'db> CallableType<'db> { CallableType::new( db, self.signatures(db).bind_self(db, self_type), - self.is_function_like(db), + self.kind(db), ) } @@ -11957,7 +12208,7 @@ impl<'db> CallableType<'db> { CallableType::new( db, self.signatures(db).apply_self(db, self_type), - self.is_function_like(db), + self.kind(db), ) } @@ -11966,7 +12217,7 @@ impl<'db> CallableType<'db> { /// Specifically, this represents a callable type with a single signature: /// `(*args: object, **kwargs: object) -> Never`. pub(crate) fn bottom(db: &'db dyn Db) -> CallableType<'db> { - Self::new(db, CallableSignature::bottom(), false) + Self::new(db, CallableSignature::bottom(), CallableTypeKind::Regular) } /// Return a "normalized" version of this `Callable` type. @@ -11976,7 +12227,7 @@ impl<'db> CallableType<'db> { CallableType::new( db, self.signatures(db).normalized_impl(db, visitor), - self.is_function_like(db), + self.kind(db), ) } @@ -11990,7 +12241,7 @@ impl<'db> CallableType<'db> { db, self.signatures(db) .recursive_type_normalized_impl(db, div, nested)?, - self.is_function_like(db), + self.kind(db), )) } @@ -12005,7 +12256,7 @@ impl<'db> CallableType<'db> { db, self.signatures(db) .apply_type_mapping_impl(db, type_mapping, tcx, visitor), - self.is_function_like(db), + self.kind(db), ) } diff --git a/crates/ty_python_semantic/src/types/call/arguments.rs b/crates/ty_python_semantic/src/types/call/arguments.rs index 6da85184ea..cc8f377271 100644 --- a/crates/ty_python_semantic/src/types/call/arguments.rs +++ b/crates/ty_python_semantic/src/types/call/arguments.rs @@ -150,6 +150,14 @@ impl<'a, 'db> CallArguments<'a, 'db> { (self.arguments.iter().copied()).zip(self.types.iter_mut()) } + /// Create a new [`CallArguments`] starting from the specified index. + pub(super) fn start_from(&self, index: usize) -> Self { + Self { + arguments: self.arguments[index..].to_vec(), + types: self.types[index..].to_vec(), + } + } + /// Returns an iterator on performing [argument type expansion]. /// /// Each element of the iterator represents a set of argument lists, where each argument list diff --git a/crates/ty_python_semantic/src/types/call/bind.rs b/crates/ty_python_semantic/src/types/call/bind.rs index da63e45208..10e9ddfca3 100644 --- a/crates/ty_python_semantic/src/types/call/bind.rs +++ b/crates/ty_python_semantic/src/types/call/bind.rs @@ -3,6 +3,7 @@ //! [signatures][crate::types::signatures], we have to handle the fact that the callable might be a //! union of types, each of which might contain multiple overloads. +use std::borrow::Cow; use std::collections::HashSet; use std::fmt; @@ -32,16 +33,15 @@ use crate::types::function::{ use crate::types::generics::{ InferableTypeVars, Specialization, SpecializationBuilder, SpecializationError, }; -use crate::types::signatures::{ - CallableSignature, Parameter, ParameterForm, ParameterKind, Parameters, -}; -use crate::types::tuple::{TupleLength, TupleType}; +use crate::types::signatures::{Parameter, ParameterForm, ParameterKind, Parameters}; +use crate::types::tuple::{TupleLength, TupleSpec, TupleType}; use crate::types::{ - BoundMethodType, BoundTypeVarIdentity, ClassLiteral, DATACLASS_FLAGS, DataclassFlags, - DataclassParams, FieldInstance, KnownBoundMethodType, KnownClass, KnownInstanceType, - MemberLookupPolicy, NominalInstanceType, PropertyInstanceType, SpecialFormType, - TrackedConstraintSet, TypeAliasType, TypeContext, TypeVarVariance, UnionBuilder, UnionType, - WrapperDescriptorKind, enums, list_members, todo_type, + BoundMethodType, BoundTypeVarIdentity, BoundTypeVarInstance, CallableSignature, + CallableTypeKind, ClassLiteral, DATACLASS_FLAGS, DataclassFlags, DataclassParams, + FieldInstance, KnownBoundMethodType, KnownClass, KnownInstanceType, MemberLookupPolicy, + NominalInstanceType, PropertyInstanceType, SpecialFormType, TrackedConstraintSet, + TypeAliasType, TypeContext, TypeVarVariance, UnionBuilder, UnionType, WrapperDescriptorKind, + enums, list_members, todo_type, }; use ruff_db::diagnostic::{Annotation, Diagnostic, SubDiagnostic, SubDiagnosticSeverity}; use ruff_python_ast::{self as ast, ArgOrKeyword, PythonVersion}; @@ -2581,20 +2581,62 @@ impl<'a, 'db> ArgumentMatcher<'a, 'db> { argument: Argument<'a>, argument_type: Option>, ) -> Result<(), ()> { - // TODO: `Type::iterate` internally handles unions, but in a lossy way. - // It might be superior here to manually map over the union and call `try_iterate` - // on each element, similar to the way that `unpacker.rs` does in the `unpack_inner` method. - // It might be a bit of a refactor, though. - // See - // for more details. --Alex - let tuple = argument_type.map(|ty| ty.iterate(db)); - let (mut argument_types, length, variable_element) = match tuple.as_ref() { - Some(tuple) => ( + enum VariadicArgumentType<'db> { + ParamSpec(Type<'db>), + Other(Cow<'db, TupleSpec<'db>>), + None, + } + + let variadic_type = match argument_type { + Some(argument_type @ Type::Union(union)) => { + // When accessing an instance attribute that is a `P.args`, the type we infer is + // `Unknown | P.args`. This needs to be special cased here to avoid calling + // `iterate` on it which will lose the `ParamSpec` information as it will return + // `object` that comes from the upper bound of `P.args`. What we want is to always + // use the `P.args` type to perform type checking against the parameter type. This + // will allow us to error when `*args: P.args` is matched against, for example, + // `n: int` and correctly type check when `*args: P.args` is matched against + // `*args: P.args` (another ParamSpec). + match union.elements(db) { + [paramspec @ Type::TypeVar(typevar), other] + | [other, paramspec @ Type::TypeVar(typevar)] + if typevar.is_paramspec(db) && other.is_unknown() => + { + VariadicArgumentType::ParamSpec(*paramspec) + } + _ => { + // TODO: Same todo comment as in the non-paramspec case below + VariadicArgumentType::Other(argument_type.iterate(db)) + } + } + } + Some(paramspec @ Type::TypeVar(typevar)) if typevar.is_paramspec(db) => { + VariadicArgumentType::ParamSpec(paramspec) + } + Some(argument_type) => { + // TODO: `Type::iterate` internally handles unions, but in a lossy way. + // It might be superior here to manually map over the union and call `try_iterate` + // on each element, similar to the way that `unpacker.rs` does in the `unpack_inner` method. + // It might be a bit of a refactor, though. + // See + // for more details. --Alex + VariadicArgumentType::Other(argument_type.iterate(db)) + } + None => VariadicArgumentType::None, + }; + + let (mut argument_types, length, variable_element) = match &variadic_type { + VariadicArgumentType::ParamSpec(paramspec) => ( + Either::Right(std::iter::empty()), + TupleLength::unknown(), + Some(*paramspec), + ), + VariadicArgumentType::Other(tuple) => ( Either::Left(tuple.all_elements().copied()), tuple.len(), tuple.variable_element().copied(), ), - None => ( + VariadicArgumentType::None => ( Either::Right(std::iter::empty()), TupleLength::unknown(), None, @@ -2669,21 +2711,39 @@ impl<'a, 'db> ArgumentMatcher<'a, 'db> { ); } } else { - let value_type = match argument_type.map(|ty| { - ty.member_lookup_with_policy( + let dunder_getitem_return_type = |ty: Type<'db>| match ty + .member_lookup_with_policy( db, Name::new_static("__getitem__"), MemberLookupPolicy::NO_INSTANCE_FALLBACK, ) .place - }) { - Some(Place::Defined(keys_method, _, Definedness::AlwaysDefined)) => keys_method + { + Place::Defined(getitem_method, _, Definedness::AlwaysDefined) => getitem_method .try_call(db, &CallArguments::positional([Type::unknown()])) .ok() .map_or_else(Type::unknown, |bindings| bindings.return_type(db)), _ => Type::unknown(), }; + let value_type = match argument_type { + Some(argument_type @ Type::Union(union)) => { + // See the comment in `match_variadic` for why we special case this situation. + match union.elements(db) { + [paramspec @ Type::TypeVar(typevar), other] + | [other, paramspec @ Type::TypeVar(typevar)] + if typevar.is_paramspec(db) && other.is_unknown() => + { + *paramspec + } + _ => dunder_getitem_return_type(argument_type), + } + } + Some(paramspec @ Type::TypeVar(typevar)) if typevar.is_paramspec(db) => paramspec, + Some(argument_type) => dunder_getitem_return_type(argument_type), + None => Type::unknown(), + }; + for (parameter_index, parameter) in self.parameters.iter().enumerate() { if self.parameter_info[parameter_index].matched && !parameter.is_keyword_variadic() { @@ -2753,6 +2813,7 @@ impl<'a, 'db> ArgumentMatcher<'a, 'db> { struct ArgumentTypeChecker<'a, 'db> { db: &'db dyn Db, + signature_type: Type<'db>, signature: &'a Signature<'db>, arguments: &'a CallArguments<'a, 'db>, argument_matches: &'a [MatchedArgument<'db>], @@ -2770,6 +2831,7 @@ impl<'a, 'db> ArgumentTypeChecker<'a, 'db> { #[expect(clippy::too_many_arguments)] fn new( db: &'db dyn Db, + signature_type: Type<'db>, signature: &'a Signature<'db>, arguments: &'a CallArguments<'a, 'db>, argument_matches: &'a [MatchedArgument<'db>], @@ -2781,6 +2843,7 @@ impl<'a, 'db> ArgumentTypeChecker<'a, 'db> { ) -> Self { Self { db, + signature_type, signature, arguments, argument_matches, @@ -3029,9 +3092,23 @@ impl<'a, 'db> ArgumentTypeChecker<'a, 'db> { } fn check_argument_types(&mut self) { + let paramspec = self + .signature + .parameters() + .find_paramspec_from_args_kwargs(self.db); + for (argument_index, adjusted_argument_index, argument, argument_type) in self.enumerate_argument_types() { + if let Some((_, paramspec)) = paramspec { + if self.try_paramspec_evaluation_at(argument_index, paramspec) { + // Once we find an argument that matches the `ParamSpec`, we can stop checking + // the remaining arguments since `ParamSpec` should always be the last + // parameter. + return; + } + } + match argument { Argument::Variadic => self.check_variadic_argument_type( argument_index, @@ -3057,6 +3134,131 @@ impl<'a, 'db> ArgumentTypeChecker<'a, 'db> { } } } + + if let Some((_, paramspec)) = paramspec { + // If we reach here, none of the arguments matched the `ParamSpec` parameter, but the + // `ParamSpec` could specialize to a parameter list containing some parameters. For + // example, + // + // ```py + // from typing import Callable + // + // def foo[**P](f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: ... + // + // def f(x: int) -> None: ... + // + // foo(f) + // ``` + // + // Here, no arguments match the `ParamSpec` parameter, but `P` specializes to `(x: int)`, + // so we need to perform a sub-call with no arguments. + self.evaluate_paramspec_sub_call(None, paramspec); + } + } + + /// Try to evaluate a `ParamSpec` sub-call at the given argument index. + /// + /// The `ParamSpec` parameter is always going to be at the end of the parameter list but there + /// can be other parameter before it. If one of these prepended positional parameters contains + /// a free `ParamSpec`, we consider that variable in scope for the purposes of extracting the + /// components of that `ParamSpec`. For example: + /// + /// ```py + /// from typing import Callable + /// + /// def foo[**P](f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: ... + /// + /// def f(x: int, y: str) -> None: ... + /// + /// foo(f, 1, "hello") # P: (x: int, y: str) + /// ``` + /// + /// Here, `P` specializes to `(x: int, y: str)` when `foo` is called with `f`, which means that + /// the parameters of `f` become a part of `foo`'s parameter list replacing the `ParamSpec` + /// parameter which is: + /// + /// ```py + /// def foo(f: Callable[[x: int, y: str], None], x: int, y: str) -> None: ... + /// ``` + /// + /// This method will check whether the parameter matching the argument at `argument_index` is + /// annotated with the components of `ParamSpec`, and if so, will invoke a sub-call considering + /// the arguments starting from `argument_index` against the specialized parameter list. + /// + /// Returns `true` if the sub-call was invoked, `false` otherwise. + fn try_paramspec_evaluation_at( + &mut self, + argument_index: usize, + paramspec: BoundTypeVarInstance<'db>, + ) -> bool { + let [parameter_index] = self.argument_matches[argument_index].parameters.as_slice() else { + return false; + }; + + if !self.signature.parameters()[*parameter_index] + .annotated_type() + .is_some_and(|ty| matches!(ty, Type::TypeVar(typevar) if typevar.is_paramspec(self.db))) + { + return false; + } + + self.evaluate_paramspec_sub_call(Some(argument_index), paramspec) + } + + /// Invoke a sub-call for the given `ParamSpec` type variable, using the remaining arguments. + /// + /// The remaining arguments start from `argument_index` if provided, otherwise no arguments + /// are passed. + /// + /// This method returns `false` if the specialization does not contain a mapping for the given + /// `paramspec`, contains an invalid mapping (i.e., not a `Callable` of kind `ParamSpecValue`) + /// or if the value is an overloaded callable. + /// + /// For more details, refer to [`Self::try_paramspec_evaluation_at`]. + fn evaluate_paramspec_sub_call( + &mut self, + argument_index: Option, + paramspec: BoundTypeVarInstance<'db>, + ) -> bool { + let Some(Type::Callable(callable)) = self + .specialization + .and_then(|specialization| specialization.get(self.db, paramspec)) + else { + return false; + }; + + if callable.kind(self.db) != CallableTypeKind::ParamSpecValue { + return false; + } + + // TODO: Support overloads? + let [signature] = callable.signatures(self.db).overloads.as_slice() else { + return false; + }; + + let sub_arguments = if let Some(argument_index) = argument_index { + self.arguments.start_from(argument_index) + } else { + CallArguments::none() + }; + + // TODO: What should be the `signature_type` here? + let bindings = match Bindings::from(Binding::single(self.signature_type, signature.clone())) + .match_parameters(self.db, &sub_arguments) + .check_types(self.db, &sub_arguments, self.call_expression_tcx, &[]) + { + Ok(bindings) => Box::new(bindings), + Err(CallError(_, bindings)) => bindings, + }; + + // SAFETY: `bindings` was created from a single binding above. + let [binding] = bindings.single_element().unwrap().overloads.as_slice() else { + unreachable!("ParamSpec sub-call should only contain a single binding"); + }; + + self.errors.extend(binding.errors.iter().cloned()); + + true } fn check_variadic_argument_type( @@ -3099,69 +3301,94 @@ impl<'a, 'db> ArgumentTypeChecker<'a, 'db> { ); } } else { - // TODO: Instead of calling the `keys` and `__getitem__` methods, we should instead - // get the constraints which satisfies the `SupportsKeysAndGetItem` protocol i.e., the - // key and value type. - let key_type = match argument_type - .member_lookup_with_policy( - self.db, - Name::new_static("keys"), - MemberLookupPolicy::NO_INSTANCE_FALLBACK, - ) - .place - { - Place::Defined(keys_method, _, Definedness::AlwaysDefined) => keys_method - .try_call(self.db, &CallArguments::none()) - .ok() - .and_then(|bindings| { - Some( - bindings - .return_type(self.db) - .try_iterate(self.db) - .ok()? - .homogeneous_element_type(self.db), + let mut value_type_fallback = |argument_type: Type<'db>| { + // TODO: Instead of calling the `keys` and `__getitem__` methods, we should + // instead get the constraints which satisfies the `SupportsKeysAndGetItem` + // protocol i.e., the key and value type. + let key_type = match argument_type + .member_lookup_with_policy( + self.db, + Name::new_static("keys"), + MemberLookupPolicy::NO_INSTANCE_FALLBACK, + ) + .place + { + Place::Defined(keys_method, _, Definedness::AlwaysDefined) => keys_method + .try_call(self.db, &CallArguments::none()) + .ok() + .and_then(|bindings| { + Some( + bindings + .return_type(self.db) + .try_iterate(self.db) + .ok()? + .homogeneous_element_type(self.db), + ) + }), + _ => None, + }; + + let Some(key_type) = key_type else { + self.errors.push(BindingError::KeywordsNotAMapping { + argument_index: adjusted_argument_index, + provided_ty: argument_type, + }); + return None; + }; + + if !key_type + .when_assignable_to( + self.db, + KnownClass::Str.to_instance(self.db), + self.inferable_typevars, + ) + .is_always_satisfied(self.db) + { + self.errors.push(BindingError::InvalidKeyType { + argument_index: adjusted_argument_index, + provided_ty: key_type, + }); + } + + Some( + match argument_type + .member_lookup_with_policy( + self.db, + Name::new_static("__getitem__"), + MemberLookupPolicy::NO_INSTANCE_FALLBACK, ) - }), - _ => None, + .place + { + Place::Defined(keys_method, _, Definedness::AlwaysDefined) => keys_method + .try_call(self.db, &CallArguments::positional([Type::unknown()])) + .ok() + .map_or_else(Type::unknown, |bindings| bindings.return_type(self.db)), + _ => Type::unknown(), + }, + ) }; - let Some(key_type) = key_type else { - self.errors.push(BindingError::KeywordsNotAMapping { - argument_index: adjusted_argument_index, - provided_ty: argument_type, - }); + let value_type = match argument_type { + Type::Union(union) => { + // See the comment in `match_variadic` for why we special case this situation. + match union.elements(self.db) { + [paramspec @ Type::TypeVar(typevar), other] + | [other, paramspec @ Type::TypeVar(typevar)] + if typevar.is_paramspec(self.db) && other.is_unknown() => + { + Some(*paramspec) + } + _ => value_type_fallback(argument_type), + } + } + Type::TypeVar(typevar) if typevar.is_paramspec(self.db) => Some(argument_type), + _ => value_type_fallback(argument_type), + }; + + let Some(value_type) = value_type else { return; }; - if !key_type - .when_assignable_to( - self.db, - KnownClass::Str.to_instance(self.db), - self.inferable_typevars, - ) - .is_always_satisfied(self.db) - { - self.errors.push(BindingError::InvalidKeyType { - argument_index: adjusted_argument_index, - provided_ty: key_type, - }); - } - - let value_type = match argument_type - .member_lookup_with_policy( - self.db, - Name::new_static("__getitem__"), - MemberLookupPolicy::NO_INSTANCE_FALLBACK, - ) - .place - { - Place::Defined(keys_method, _, Definedness::AlwaysDefined) => keys_method - .try_call(self.db, &CallArguments::positional([Type::unknown()])) - .ok() - .map_or_else(Type::unknown, |bindings| bindings.return_type(self.db)), - _ => Type::unknown(), - }; - for (argument_type, parameter_index) in std::iter::repeat(value_type).zip(&self.argument_matches[argument_index].parameters) { @@ -3339,6 +3566,7 @@ impl<'db> Binding<'db> { ) { let mut checker = ArgumentTypeChecker::new( db, + self.signature_type, &self.signature, arguments, &self.argument_matches, diff --git a/crates/ty_python_semantic/src/types/class.rs b/crates/ty_python_semantic/src/types/class.rs index 00b5bed3ec..514b312313 100644 --- a/crates/ty_python_semantic/src/types/class.rs +++ b/crates/ty_python_semantic/src/types/class.rs @@ -32,12 +32,13 @@ use crate::types::tuple::{TupleSpec, TupleType}; use crate::types::typed_dict::typed_dict_params_from_class_def; use crate::types::visitor::{TypeCollector, TypeVisitor, walk_type_with_recursion_guard}; use crate::types::{ - ApplyTypeMappingVisitor, Binding, BoundSuperType, CallableType, CallableTypes, DATACLASS_FLAGS, - DataclassFlags, DataclassParams, DeprecatedInstance, FindLegacyTypeVarsVisitor, - HasRelationToVisitor, IsDisjointVisitor, IsEquivalentVisitor, KnownInstanceType, - ManualPEP695TypeAliasType, MaterializationKind, NormalizedVisitor, PropertyInstanceType, - StringLiteralType, TypeAliasType, TypeContext, TypeMapping, TypeRelation, TypedDictParams, - UnionBuilder, VarianceInferable, binding_type, declaration_type, determine_upper_bound, + ApplyTypeMappingVisitor, Binding, BoundSuperType, CallableType, CallableTypeKind, + CallableTypes, DATACLASS_FLAGS, DataclassFlags, DataclassParams, DeprecatedInstance, + FindLegacyTypeVarsVisitor, HasRelationToVisitor, IsDisjointVisitor, IsEquivalentVisitor, + KnownInstanceType, ManualPEP695TypeAliasType, MaterializationKind, NormalizedVisitor, + PropertyInstanceType, StringLiteralType, TypeAliasType, TypeContext, TypeMapping, TypeRelation, + TypedDictParams, UnionBuilder, VarianceInferable, binding_type, declaration_type, + determine_upper_bound, }; use crate::{ Db, FxIndexMap, FxIndexSet, FxOrderSet, Program, @@ -1021,8 +1022,11 @@ impl<'db> ClassType<'db> { let getitem_signature = CallableSignature::from_overloads(overload_signatures); - let getitem_type = - Type::Callable(CallableType::new(db, getitem_signature, true)); + let getitem_type = Type::Callable(CallableType::new( + db, + getitem_signature, + CallableTypeKind::FunctionLike, + )); Member::definitely_declared(getitem_type) }) .unwrap_or_else(fallback_member_lookup) @@ -1188,7 +1192,7 @@ impl<'db> ClassType<'db> { let dunder_new_bound_method = CallableType::new( db, dunder_new_signature.bind_self(db, Some(instance_ty)), - true, + CallableTypeKind::FunctionLike, ); if returns_non_subclass { @@ -1257,7 +1261,7 @@ impl<'db> ClassType<'db> { Some(CallableType::new( db, synthesized_dunder_init_signature, - true, + CallableTypeKind::FunctionLike, )) } else { None @@ -2080,9 +2084,11 @@ impl<'db> ClassLiteral<'db> { ) -> PlaceAndQualifiers<'db> { fn into_function_like_callable<'d>(db: &'d dyn Db, ty: Type<'d>) -> Type<'d> { match ty { - Type::Callable(callable_ty) => { - Type::Callable(CallableType::new(db, callable_ty.signatures(db), true)) - } + Type::Callable(callable_ty) => Type::Callable(CallableType::new( + db, + callable_ty.signatures(db), + CallableTypeKind::FunctionLike, + )), Type::Union(union) => { union.map(db, |element| into_function_like_callable(db, *element)) } @@ -2677,7 +2683,7 @@ impl<'db> ClassLiteral<'db> { ), Some(Type::none(db)), )), - true, + CallableTypeKind::FunctionLike, ))); } @@ -2703,7 +2709,7 @@ impl<'db> ClassLiteral<'db> { Some(Type::Callable(CallableType::new( db, CallableSignature::from_overloads(overloads), - true, + CallableTypeKind::FunctionLike, ))) } (CodeGeneratorKind::TypedDict, "__getitem__") => { @@ -2730,7 +2736,7 @@ impl<'db> ClassLiteral<'db> { Some(Type::Callable(CallableType::new( db, CallableSignature::from_overloads(overloads), - true, + CallableTypeKind::FunctionLike, ))) } (CodeGeneratorKind::TypedDict, "get") => { @@ -2838,7 +2844,7 @@ impl<'db> ClassLiteral<'db> { Some(Type::Callable(CallableType::new( db, CallableSignature::from_overloads(overloads), - true, + CallableTypeKind::FunctionLike, ))) } (CodeGeneratorKind::TypedDict, "pop") => { @@ -2898,7 +2904,7 @@ impl<'db> ClassLiteral<'db> { Some(Type::Callable(CallableType::new( db, CallableSignature::from_overloads(overloads), - true, + CallableTypeKind::FunctionLike, ))) } (CodeGeneratorKind::TypedDict, "setdefault") => { @@ -2926,7 +2932,7 @@ impl<'db> ClassLiteral<'db> { Some(Type::Callable(CallableType::new( db, CallableSignature::from_overloads(overloads), - true, + CallableTypeKind::FunctionLike, ))) } (CodeGeneratorKind::TypedDict, "update") => { diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 95ee3ad748..19ed71bbff 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -17,13 +17,16 @@ use crate::Db; use crate::types::class::{ClassLiteral, ClassType, GenericAlias}; use crate::types::function::{FunctionType, OverloadLiteral}; use crate::types::generics::{GenericContext, Specialization}; -use crate::types::signatures::{CallableSignature, Parameter, Parameters, Signature}; +use crate::types::signatures::{ + CallableSignature, Parameter, Parameters, ParametersKind, Signature, +}; use crate::types::tuple::TupleSpec; use crate::types::visitor::TypeVisitor; use crate::types::{ - BoundTypeVarIdentity, CallableType, IntersectionType, KnownBoundMethodType, KnownClass, - KnownInstanceType, MaterializationKind, Protocol, ProtocolInstanceType, SpecialFormType, - StringLiteralType, SubclassOfInner, Type, UnionType, WrapperDescriptorKind, visitor, + BoundTypeVarIdentity, CallableType, CallableTypeKind, IntersectionType, KnownBoundMethodType, + KnownClass, KnownInstanceType, MaterializationKind, Protocol, ProtocolInstanceType, + SpecialFormType, StringLiteralType, SubclassOfInner, Type, UnionType, WrapperDescriptorKind, + visitor, }; /// Settings for displaying types and signatures @@ -937,6 +940,9 @@ impl Display for DisplayBoundTypeVarIdentity<'_> { if let Some(binding_context) = self.bound_typevar_identity.binding_context.name(self.db) { write!(f, "@{binding_context}")?; } + if let Some(paramspec_attr) = self.bound_typevar_identity.paramspec_attr { + write!(f, ".{paramspec_attr}")?; + } Ok(()) } } @@ -1298,7 +1304,12 @@ impl<'db> DisplayGenericContext<'_, 'db> { f.write_str(", ")?; } f.set_invalid_syntax(); - f.write_str(bound_typevar.typevar(self.db).name(self.db))?; + let typevar = bound_typevar.typevar(self.db); + if typevar.is_paramspec(self.db) { + write!(f, "**{}", typevar.name(self.db))?; + } else { + f.write_str(typevar.name(self.db))?; + } } f.write_char(']') } @@ -1459,6 +1470,7 @@ impl<'db> CallableType<'db> { ) -> DisplayCallableType<'a, 'db> { DisplayCallableType { signatures: self.signatures(db), + kind: self.kind(db), db, settings, } @@ -1467,6 +1479,7 @@ impl<'db> CallableType<'db> { pub(crate) struct DisplayCallableType<'a, 'db> { signatures: &'a CallableSignature<'db>, + kind: CallableTypeKind, db: &'db dyn Db, settings: DisplaySettings<'db>, } @@ -1474,9 +1487,18 @@ pub(crate) struct DisplayCallableType<'a, 'db> { impl<'db> FmtDetailed<'db> for DisplayCallableType<'_, 'db> { fn fmt_detailed(&self, f: &mut TypeWriter<'_, '_, 'db>) -> fmt::Result { match self.signatures.overloads.as_slice() { - [signature] => signature - .display_with(self.db, self.settings.clone()) - .fmt_detailed(f), + [signature] => { + if matches!(self.kind, CallableTypeKind::ParamSpecValue) { + signature + .parameters() + .display_with(self.db, self.settings.clone()) + .fmt_detailed(f) + } else { + signature + .display_with(self.db, self.settings.clone()) + .fmt_detailed(f) + } + } signatures => { // TODO: How to display overloads? if !self.settings.multiline { @@ -1552,73 +1574,11 @@ impl<'db> FmtDetailed<'db> for DisplaySignature<'_, 'db> { f.set_invalid_syntax(); // When we exit this function, write a marker signaling we're ending a signature let mut f = f.with_detail(TypeDetail::SignatureEnd); - let multiline = self.settings.multiline && self.parameters.len() > 1; - // Opening parenthesis - f.write_char('(')?; - if multiline { - f.write_str("\n ")?; - } - if self.parameters.is_gradual() { - // We represent gradual form as `...` in the signature, internally the parameters still - // contain `(*args, **kwargs)` parameters. - f.write_str("...")?; - } else { - let mut star_added = false; - let mut needs_slash = false; - let mut first = true; - let arg_separator = if multiline { ",\n " } else { ", " }; - for parameter in self.parameters.as_slice() { - // Handle special separators - if !star_added && parameter.is_keyword_only() { - if !first { - f.write_str(arg_separator)?; - } - f.write_char('*')?; - star_added = true; - first = false; - } - if parameter.is_positional_only() { - needs_slash = true; - } else if needs_slash { - if !first { - f.write_str(arg_separator)?; - } - f.write_char('/')?; - needs_slash = false; - first = false; - } - - // Add comma before parameter if not first - if !first { - f.write_str(arg_separator)?; - } - - // Write parameter with range tracking - let param_name = parameter - .display_name() - .map(|name| name.to_string()) - .unwrap_or_default(); - parameter - .display_with(self.db, self.settings.singleline()) - .fmt_detailed(&mut f.with_detail(TypeDetail::Parameter(param_name)))?; - - first = false; - } - - if needs_slash { - if !first { - f.write_str(arg_separator)?; - } - f.write_char('/')?; - } - } - - if multiline { - f.write_char('\n')?; - } - // Closing parenthesis - f.write_char(')')?; + // Parameters + self.parameters + .display_with(self.db, self.settings.clone()) + .fmt_detailed(&mut f)?; // Return type let return_ty = self.return_ty.unwrap_or_else(Type::unknown); @@ -1646,6 +1606,120 @@ pub(crate) struct SignatureDisplayDetails { pub parameter_names: Vec, } +impl<'db> Parameters<'db> { + fn display_with<'a>( + &'a self, + db: &'db dyn Db, + settings: DisplaySettings<'db>, + ) -> DisplayParameters<'a, 'db> { + DisplayParameters { + parameters: self, + db, + settings, + } + } +} + +struct DisplayParameters<'a, 'db> { + parameters: &'a Parameters<'db>, + db: &'db dyn Db, + settings: DisplaySettings<'db>, +} + +impl<'db> FmtDetailed<'db> for DisplayParameters<'_, 'db> { + fn fmt_detailed(&self, f: &mut TypeWriter<'_, '_, 'db>) -> fmt::Result { + // For `ParamSpec` kind, the parameters still contain `*args` and `**kwargs`, but we + // display them as `**P` instead, so avoid multiline in that case. + // TODO: This might change once we support `Concatenate` + let multiline = self.settings.multiline + && self.parameters.len() > 1 + && !matches!( + self.parameters.kind(), + ParametersKind::Gradual | ParametersKind::ParamSpec(_) + ); + // Opening parenthesis + f.write_char('(')?; + if multiline { + f.write_str("\n ")?; + } + match self.parameters.kind() { + ParametersKind::Standard => { + let mut star_added = false; + let mut needs_slash = false; + let mut first = true; + let arg_separator = if multiline { ",\n " } else { ", " }; + + for parameter in self.parameters.as_slice() { + // Handle special separators + if !star_added && parameter.is_keyword_only() { + if !first { + f.write_str(arg_separator)?; + } + f.write_char('*')?; + star_added = true; + first = false; + } + if parameter.is_positional_only() { + needs_slash = true; + } else if needs_slash { + if !first { + f.write_str(arg_separator)?; + } + f.write_char('/')?; + needs_slash = false; + first = false; + } + + // Add comma before parameter if not first + if !first { + f.write_str(arg_separator)?; + } + + // Write parameter with range tracking + let param_name = parameter + .display_name() + .map(|name| name.to_string()) + .unwrap_or_default(); + parameter + .display_with(self.db, self.settings.singleline()) + .fmt_detailed(&mut f.with_detail(TypeDetail::Parameter(param_name)))?; + + first = false; + } + + if needs_slash { + if !first { + f.write_str(arg_separator)?; + } + f.write_char('/')?; + } + } + ParametersKind::Gradual => { + // We represent gradual form as `...` in the signature, internally the parameters still + // contain `(*args, **kwargs)` parameters. + f.write_str("...")?; + } + ParametersKind::ParamSpec(typevar) => { + write!(f, "**{}", typevar.name(self.db))?; + if let Some(name) = typevar.binding_context(self.db).name(self.db) { + write!(f, "@{name}")?; + } + } + } + if multiline { + f.write_char('\n')?; + } + // Closing parenthesis + f.write_char(')') + } +} + +impl Display for DisplayParameters<'_, '_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.fmt_detailed(&mut TypeWriter::Formatter(f)) + } +} + impl<'db> Parameter<'db> { fn display_with<'a>( &'a self, diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index 7380ec86f3..6b6c798615 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -79,8 +79,8 @@ use crate::types::narrow::ClassInfoConstraintFunction; use crate::types::signatures::{CallableSignature, Signature}; use crate::types::visitor::any_over_type; use crate::types::{ - ApplyTypeMappingVisitor, BoundMethodType, BoundTypeVarInstance, CallableType, ClassBase, - ClassLiteral, ClassType, DeprecatedInstance, DynamicType, FindLegacyTypeVarsVisitor, + ApplyTypeMappingVisitor, BoundMethodType, BoundTypeVarInstance, CallableType, CallableTypeKind, + ClassBase, ClassLiteral, ClassType, DeprecatedInstance, DynamicType, FindLegacyTypeVarsVisitor, HasRelationToVisitor, IsDisjointVisitor, IsEquivalentVisitor, KnownClass, KnownInstanceType, NormalizedVisitor, SpecialFormType, Truthiness, Type, TypeContext, TypeMapping, TypeRelation, UnionBuilder, binding_type, definition_expression_type, walk_signature, @@ -1007,7 +1007,7 @@ impl<'db> FunctionType<'db> { /// Convert the `FunctionType` into a [`CallableType`]. pub(crate) fn into_callable_type(self, db: &'db dyn Db) -> CallableType<'db> { - CallableType::new(db, self.signature(db), true) + CallableType::new(db, self.signature(db), CallableTypeKind::FunctionLike) } /// Convert the `FunctionType` into a [`BoundMethodType`]. diff --git a/crates/ty_python_semantic/src/types/generics.rs b/crates/ty_python_semantic/src/types/generics.rs index 432785b778..7db5f7e7a2 100644 --- a/crates/ty_python_semantic/src/types/generics.rs +++ b/crates/ty_python_semantic/src/types/generics.rs @@ -13,14 +13,15 @@ use crate::types::class::ClassType; use crate::types::class_base::ClassBase; use crate::types::constraints::ConstraintSet; use crate::types::instance::{Protocol, ProtocolInstanceType}; -use crate::types::signatures::Parameters; +use crate::types::signatures::{Parameters, ParametersKind}; use crate::types::tuple::{TupleSpec, TupleType, walk_tuple_type}; use crate::types::visitor::{TypeCollector, TypeVisitor, walk_type_with_recursion_guard}; use crate::types::{ ApplyTypeMappingVisitor, BindingContext, BoundTypeVarIdentity, BoundTypeVarInstance, - ClassLiteral, FindLegacyTypeVarsVisitor, HasRelationToVisitor, IsDisjointVisitor, - IsEquivalentVisitor, KnownClass, KnownInstanceType, MaterializationKind, NormalizedVisitor, - Type, TypeContext, TypeMapping, TypeRelation, TypeVarBoundOrConstraints, TypeVarIdentity, + CallableSignature, CallableType, CallableTypeKind, CallableTypes, ClassLiteral, + FindLegacyTypeVarsVisitor, HasRelationToVisitor, IsDisjointVisitor, IsEquivalentVisitor, + KnownClass, KnownInstanceType, MaterializationKind, NormalizedVisitor, Signature, Type, + TypeContext, TypeMapping, TypeRelation, TypeVarBoundOrConstraints, TypeVarIdentity, TypeVarInstance, TypeVarKind, TypeVarVariance, UnionType, declaration_type, walk_bound_type_var_type, }; @@ -347,6 +348,21 @@ impl<'db> GenericContext<'db> { self.variables_inner(db).values().copied() } + /// Returns `true` if this generic context contains exactly one `ParamSpec` and no other type + /// variables. + /// + /// For example: + /// ```py + /// class Foo[**P]: ... # true + /// class Bar[T, **P]: ... # false + /// class Baz[T]: ... # false + /// ``` + pub(crate) fn exactly_one_paramspec(self, db: &'db dyn Db) -> bool { + self.variables(db) + .exactly_one() + .is_ok_and(|bound_typevar| bound_typevar.is_paramspec(db)) + } + fn variable_from_type_param( db: &'db dyn Db, index: &'db SemanticIndex<'db>, @@ -363,8 +379,16 @@ impl<'db> GenericContext<'db> { }; Some(typevar.with_binding_context(db, binding_context)) } - // TODO: Support these! - ast::TypeParam::ParamSpec(_) => None, + ast::TypeParam::ParamSpec(node) => { + let definition = index.expect_single_definition(node); + let Type::KnownInstance(KnownInstanceType::TypeVar(typevar)) = + declaration_type(db, definition).inner_type() + else { + return None; + }; + Some(typevar.with_binding_context(db, binding_context)) + } + // TODO: Support this! ast::TypeParam::TypeVarTuple(_) => None, } } @@ -578,7 +602,15 @@ impl<'db> GenericContext<'db> { // // If there is a mapping for `T`, we want to map `U` to that type, not to `T`. To handle // this, we repeatedly apply the specialization to itself, until we reach a fixed point. - let mut expanded = vec![Type::unknown(); types.len()]; + let mut expanded = Vec::with_capacity(types.len()); + for typevar in variables.clone() { + if typevar.is_paramspec(db) { + expanded.push(Type::paramspec_value_callable(db, Parameters::unknown())); + } else { + expanded.push(Type::unknown()); + } + } + for (idx, (ty, typevar)) in types.zip(variables).enumerate() { if let Some(ty) = ty { expanded[idx] = ty; @@ -1409,6 +1441,15 @@ impl<'db> SpecializationBuilder<'db> { match self.types.entry(identity) { Entry::Occupied(mut entry) => { + // TODO: The spec says that when a ParamSpec is used multiple times in a signature, + // the type checker can solve it to a common behavioral supertype. We don't + // implement that yet so in case there are multiple ParamSpecs, use the + // specialization from the first occurrence. + // https://github.com/astral-sh/ty/issues/1778 + // https://github.com/astral-sh/ruff/pull/21445#discussion_r2591510145 + if bound_typevar.is_paramspec(self.db) { + return; + } *entry.get_mut() = UnionType::from_elements(self.db, [*entry.get(), ty]); } Entry::Vacant(entry) => { @@ -1669,6 +1710,47 @@ impl<'db> SpecializationBuilder<'db> { } } + (Type::Callable(formal_callable), _) => { + if let Some(actual_callable) = actual + .try_upcast_to_callable(self.db) + .and_then(CallableTypes::exactly_one) + { + // We're only interested in a formal callable of the form `Callable[P, ...]` for + // now where `P` is a `ParamSpec`. + // TODO: This would need to be updated once we support `Concatenate` + // TODO: What to do for overloaded callables? + let [signature] = formal_callable.signatures(self.db).as_slice() else { + return Ok(()); + }; + let formal_parameters = signature.parameters(); + let ParametersKind::ParamSpec(typevar) = formal_parameters.kind() else { + return Ok(()); + }; + let paramspec_value = match actual_callable.signatures(self.db).as_slice() { + [] => return Ok(()), + [actual_signature] => match actual_signature.parameters().kind() { + ParametersKind::ParamSpec(typevar) => Type::TypeVar(typevar), + _ => Type::Callable(CallableType::new( + self.db, + CallableSignature::single(Signature::new( + actual_signature.parameters().clone(), + None, + )), + CallableTypeKind::ParamSpecValue, + )), + }, + actual_signatures => Type::Callable(CallableType::new( + self.db, + CallableSignature::from_overloads(actual_signatures.iter().map( + |signature| Signature::new(signature.parameters().clone(), None), + )), + CallableTypeKind::ParamSpecValue, + )), + }; + self.add_type_mapping(typevar, paramspec_value, polarity, &mut f); + } + } + // TODO: Add more forms that we can structurally induct into: type[C], callables _ => {} } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index b33883d234..6ddaea40a4 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -56,17 +56,17 @@ use crate::types::class::{CodeGeneratorKind, FieldKind, MetaclassErrorKind, Meth use crate::types::context::{InNoTypeCheck, InferContext}; use crate::types::cyclic::CycleDetector; use crate::types::diagnostic::{ - CALL_NON_CALLABLE, CONFLICTING_DECLARATIONS, CONFLICTING_METACLASS, CYCLIC_CLASS_DEFINITION, - CYCLIC_TYPE_ALIAS_DEFINITION, DIVISION_BY_ZERO, DUPLICATE_KW_ONLY, INCONSISTENT_MRO, - INVALID_ARGUMENT_TYPE, INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, INVALID_BASE, - INVALID_DECLARATION, INVALID_GENERIC_CLASS, INVALID_KEY, INVALID_LEGACY_TYPE_VARIABLE, - INVALID_METACLASS, INVALID_NAMED_TUPLE, INVALID_NEWTYPE, INVALID_OVERLOAD, - INVALID_PARAMETER_DEFAULT, INVALID_PARAMSPEC, INVALID_PROTOCOL, INVALID_TYPE_ARGUMENTS, - INVALID_TYPE_FORM, INVALID_TYPE_GUARD_CALL, INVALID_TYPE_VARIABLE_CONSTRAINTS, - IncompatibleBases, NON_SUBSCRIPTABLE, POSSIBLY_MISSING_ATTRIBUTE, - POSSIBLY_MISSING_IMPLICIT_CALL, POSSIBLY_MISSING_IMPORT, SUBCLASS_OF_FINAL_CLASS, - UNDEFINED_REVEAL, UNRESOLVED_ATTRIBUTE, UNRESOLVED_GLOBAL, UNRESOLVED_IMPORT, - UNRESOLVED_REFERENCE, UNSUPPORTED_OPERATOR, USELESS_OVERLOAD_BODY, + self, CALL_NON_CALLABLE, CONFLICTING_DECLARATIONS, CONFLICTING_METACLASS, + CYCLIC_CLASS_DEFINITION, CYCLIC_TYPE_ALIAS_DEFINITION, DIVISION_BY_ZERO, DUPLICATE_KW_ONLY, + INCONSISTENT_MRO, INVALID_ARGUMENT_TYPE, INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, + INVALID_BASE, INVALID_DECLARATION, INVALID_GENERIC_CLASS, INVALID_KEY, + INVALID_LEGACY_TYPE_VARIABLE, INVALID_METACLASS, INVALID_NAMED_TUPLE, INVALID_NEWTYPE, + INVALID_OVERLOAD, INVALID_PARAMETER_DEFAULT, INVALID_PARAMSPEC, INVALID_PROTOCOL, + INVALID_TYPE_ARGUMENTS, INVALID_TYPE_FORM, INVALID_TYPE_GUARD_CALL, + INVALID_TYPE_VARIABLE_CONSTRAINTS, IncompatibleBases, NON_SUBSCRIPTABLE, + POSSIBLY_MISSING_ATTRIBUTE, POSSIBLY_MISSING_IMPLICIT_CALL, POSSIBLY_MISSING_IMPORT, + SUBCLASS_OF_FINAL_CLASS, UNDEFINED_REVEAL, UNRESOLVED_ATTRIBUTE, UNRESOLVED_GLOBAL, + UNRESOLVED_IMPORT, UNRESOLVED_REFERENCE, UNSUPPORTED_OPERATOR, USELESS_OVERLOAD_BODY, hint_if_stdlib_attribute_exists_on_other_versions, hint_if_stdlib_submodule_exists_on_other_versions, report_attempted_protocol_instantiation, report_bad_dunder_set_call, report_cannot_pop_required_field_on_typed_dict, @@ -94,7 +94,6 @@ use crate::types::infer::nearest_enclosing_function; use crate::types::instance::SliceLiteral; use crate::types::mro::MroErrorKind; use crate::types::newtype::NewType; -use crate::types::signatures::{Parameter, Parameters, Signature}; use crate::types::subclass_of::SubclassOfInner; use crate::types::tuple::{Tuple, TupleLength, TupleSpec, TupleType}; use crate::types::typed_dict::{ @@ -103,16 +102,17 @@ use crate::types::typed_dict::{ }; use crate::types::visitor::any_over_type; use crate::types::{ - BoundTypeVarInstance, CallDunderError, CallableBinding, CallableType, CallableTypes, + BoundTypeVarInstance, CallDunderError, CallableBinding, CallableType, CallableTypeKind, ClassLiteral, ClassType, DataclassParams, DynamicType, InternedType, IntersectionBuilder, IntersectionType, KnownClass, KnownInstanceType, LintDiagnosticGuard, MemberLookupPolicy, - MetaclassCandidate, PEP695TypeAliasType, ParameterForm, SpecialFormType, SubclassOfType, - TrackedConstraintSet, Truthiness, Type, TypeAliasType, TypeAndQualifiers, TypeContext, - TypeQualifiers, TypeVarBoundOrConstraints, TypeVarBoundOrConstraintsEvaluation, - TypeVarDefaultEvaluation, TypeVarIdentity, TypeVarInstance, TypeVarKind, TypeVarVariance, - TypedDictType, UnionBuilder, UnionType, UnionTypeInstance, binding_type, infer_scope_types, - overrides, todo_type, + MetaclassCandidate, PEP695TypeAliasType, ParamSpecAttrKind, Parameter, ParameterForm, + Parameters, Signature, SpecialFormType, SubclassOfType, TrackedConstraintSet, Truthiness, Type, + TypeAliasType, TypeAndQualifiers, TypeContext, TypeQualifiers, TypeVarBoundOrConstraints, + TypeVarBoundOrConstraintsEvaluation, TypeVarDefaultEvaluation, TypeVarIdentity, + TypeVarInstance, TypeVarKind, TypeVarVariance, TypedDictType, UnionBuilder, UnionType, + UnionTypeInstance, binding_type, infer_scope_types, todo_type, }; +use crate::types::{CallableTypes, overrides}; use crate::types::{ClassBase, add_inferred_python_version_hint_to_diagnostic}; use crate::unpack::{EvaluationMode, UnpackPosition}; use crate::{Db, FxIndexSet, FxOrderSet, Program}; @@ -2347,7 +2347,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Type::Callable(callable) => Some(Type::Callable(CallableType::new( db, callable.signatures(db), - true, + CallableTypeKind::FunctionLike, ))), Type::Union(union) => union .try_map(db, |element| into_function_like_callable(db, *element)), @@ -2612,7 +2612,41 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { todo_type!("PEP 646") } else { let annotated_type = self.file_expression_type(annotation); - Type::homogeneous_tuple(self.db(), annotated_type) + if let Type::TypeVar(typevar) = annotated_type + && typevar.is_paramspec(self.db()) + { + match typevar.paramspec_attr(self.db()) { + // `*args: P.args` + Some(ParamSpecAttrKind::Args) => annotated_type, + + // `*args: P.kwargs` + Some(ParamSpecAttrKind::Kwargs) => { + // TODO: Should this diagnostic be raised as part of + // `ArgumentTypeChecker`? + if let Some(builder) = + self.context.report_lint(&INVALID_TYPE_FORM, annotation) + { + let name = typevar.name(self.db()); + let mut diag = builder.into_diagnostic(format_args!( + "`{name}.kwargs` is valid only in `**kwargs` annotation", + )); + diag.set_primary_message(format_args!( + "Did you mean `{name}.args`?" + )); + diagnostic::add_type_expression_reference_link(diag); + } + Type::homogeneous_tuple(self.db(), Type::unknown()) + } + + // `*args: P` + None => { + // The diagnostic for this case is handled in `in_type_expression`. + Type::homogeneous_tuple(self.db(), Type::unknown()) + } + } + } else { + Type::homogeneous_tuple(self.db(), annotated_type) + } }; self.add_declaration_with_binding( @@ -2695,7 +2729,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { typing_self(db, self.scope(), Some(method_definition), class_literal) } - /// Set initial declared/inferred types for a `*args` variadic positional parameter. + /// Set initial declared/inferred types for a `**kwargs` keyword-variadic parameter. /// /// The annotated type is implicitly wrapped in a string-keyed dictionary. /// @@ -2708,11 +2742,48 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { definition: Definition<'db>, ) { if let Some(annotation) = parameter.annotation() { - let annotated_ty = self.file_expression_type(annotation); - let ty = KnownClass::Dict.to_specialized_instance( - self.db(), - [KnownClass::Str.to_instance(self.db()), annotated_ty], - ); + let annotated_type = self.file_expression_type(annotation); + let ty = if let Type::TypeVar(typevar) = annotated_type + && typevar.is_paramspec(self.db()) + { + match typevar.paramspec_attr(self.db()) { + // `**kwargs: P.args` + Some(ParamSpecAttrKind::Args) => { + // TODO: Should this diagnostic be raised as part of `ArgumentTypeChecker`? + if let Some(builder) = + self.context.report_lint(&INVALID_TYPE_FORM, annotation) + { + let name = typevar.name(self.db()); + let mut diag = builder.into_diagnostic(format_args!( + "`{name}.args` is valid only in `*args` annotation", + )); + diag.set_primary_message(format_args!("Did you mean `{name}.kwargs`?")); + diagnostic::add_type_expression_reference_link(diag); + } + KnownClass::Dict.to_specialized_instance( + self.db(), + [KnownClass::Str.to_instance(self.db()), Type::unknown()], + ) + } + + // `**kwargs: P.kwargs` + Some(ParamSpecAttrKind::Kwargs) => annotated_type, + + // `**kwargs: P` + None => { + // The diagnostic for this case is handled in `in_type_expression`. + KnownClass::Dict.to_specialized_instance( + self.db(), + [KnownClass::Str.to_instance(self.db()), Type::unknown()], + ) + } + } + } else { + KnownClass::Dict.to_specialized_instance( + self.db(), + [KnownClass::Str.to_instance(self.db()), annotated_type], + ) + }; self.add_declaration_with_binding( parameter.into(), definition, @@ -3337,20 +3408,81 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { }; let previous_deferred_state = std::mem::replace(&mut self.deferred_state, DeferredExpressionState::Deferred); - let default_ty = self.infer_paramspec_default(default); - self.store_expression_type(default, default_ty); + self.infer_paramspec_default(default); self.deferred_state = previous_deferred_state; } - fn infer_paramspec_default(&mut self, default: &ast::Expr) -> Type<'db> { - // This is the same logic as `TypeInferenceBuilder::infer_callable_parameter_types` except - // for the subscript branch which is required for `Concatenate` but that cannot be - // specified in this context. - match default { - ast::Expr::EllipsisLiteral(_) => { - Type::single_callable(self.db(), Signature::new(Parameters::gradual_form(), None)) + fn infer_paramspec_default(&mut self, default_expr: &ast::Expr) { + match default_expr { + ast::Expr::EllipsisLiteral(ellipsis) => { + let ty = self.infer_ellipsis_literal_expression(ellipsis); + self.store_expression_type(default_expr, ty); + return; } ast::Expr::List(ast::ExprList { elts, .. }) => { + let types = elts + .iter() + .map(|elt| self.infer_type_expression(elt)) + .collect::>(); + // N.B. We cannot represent a heterogeneous list of types in our type system, so we + // use a heterogeneous tuple type to represent the list of types instead. + self.store_expression_type( + default_expr, + Type::heterogeneous_tuple(self.db(), types), + ); + return; + } + ast::Expr::Name(_) => { + let ty = self.infer_type_expression(default_expr); + let is_paramspec = match ty { + Type::TypeVar(typevar) => typevar.is_paramspec(self.db()), + Type::KnownInstance(known_instance) => { + known_instance.class(self.db()) == KnownClass::ParamSpec + } + _ => false, + }; + if is_paramspec { + return; + } + } + _ => {} + } + if let Some(builder) = self.context.report_lint(&INVALID_PARAMSPEC, default_expr) { + builder.into_diagnostic( + "The default value to `ParamSpec` must be either \ + a list of types, `ParamSpec`, or `...`", + ); + } + } + + /// Infer the type of the expression that represents an explicit specialization of a + /// `ParamSpec` type variable. + fn infer_paramspec_explicit_specialization_value( + &mut self, + expr: &ast::Expr, + exactly_one_paramspec: bool, + ) -> Result, ()> { + let db = self.db(); + + match expr { + ast::Expr::EllipsisLiteral(_) => { + return Ok(Type::paramspec_value_callable( + db, + Parameters::gradual_form(), + )); + } + + ast::Expr::Tuple(ast::ExprTuple { elts, .. }) + | ast::Expr::List(ast::ExprList { elts, .. }) => { + // This should be taken care of by the caller. + if expr.is_tuple_expr() { + assert!( + exactly_one_paramspec, + "Inferring ParamSpec value during explicit specialization for a \ + tuple expression should only happen when it contains exactly one ParamSpec" + ); + } + let mut parameter_types = Vec::with_capacity(elts.len()); // Whether to infer `Todo` for the parameters @@ -3379,41 +3511,109 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ) }; - Type::single_callable(self.db(), Signature::new(parameters, None)) + return Ok(Type::paramspec_value_callable(db, parameters)); } - ast::Expr::Name(name) => { - let name_ty = self.infer_name_load(name); - let is_paramspec = match name_ty { - Type::KnownInstance(known_instance) => { - known_instance.class(self.db()) == KnownClass::ParamSpec + + ast::Expr::Subscript(_) => { + // TODO: Support `Concatenate[...]` + return Ok(Type::paramspec_value_callable(db, Parameters::todo())); + } + + ast::Expr::Name(_) => { + let param_type = self.infer_type_expression(expr); + + match param_type { + Type::TypeVar(typevar) if typevar.is_paramspec(db) => { + return Ok(param_type); } - Type::NominalInstance(nominal) => { - nominal.has_known_class(self.db(), KnownClass::ParamSpec) + + Type::KnownInstance(known_instance) + if known_instance.class(self.db()) == KnownClass::ParamSpec => + { + // TODO: Emit diagnostic: "ParamSpec "P" is unbound" + return Err(()); } - _ => false, - }; - if is_paramspec { - name_ty - } else { - if let Some(builder) = self.context.report_lint(&INVALID_PARAMSPEC, default) { - builder.into_diagnostic( - "The default value to `ParamSpec` must be either a list of types, \ - `ParamSpec`, or `...`", - ); + + // This is to handle the following case: + // + // ```python + // from typing import ParamSpec + // + // class Foo[**P]: ... + // + // Foo[ParamSpec] # P: (ParamSpec, /) + // ``` + Type::NominalInstance(nominal) + if nominal.has_known_class(self.db(), KnownClass::ParamSpec) => + { + return Ok(Type::paramspec_value_callable( + db, + Parameters::new( + self.db(), + [ + Parameter::positional_only(None) + .with_annotated_type(param_type), + ], + ), + )); } - Type::unknown() + + _ if exactly_one_paramspec => { + // Square brackets are optional when `ParamSpec` is the only type variable + // being specialized. This means that a single name expression represents a + // parameter list with a single parameter. For example, + // + // ```python + // class OnlyParamSpec[**P]: ... + // + // OnlyParamSpec[int] # P: (int, /) + // ``` + let parameters = + if param_type.is_todo() { + Parameters::todo() + } else { + Parameters::new( + self.db(), + [Parameter::positional_only(None) + .with_annotated_type(param_type)], + ) + }; + return Ok(Type::paramspec_value_callable(db, parameters)); + } + + // This is specifically to handle a case where there are more than one type + // variables and at least one of them is a `ParamSpec` which is specialized + // using `typing.Any`. This isn't explicitly allowed in the spec, but both mypy + // and Pyright allows this and the ecosystem report suggested there are usages + // of this in the wild e.g., `staticmethod[Any, Any]`. For example, + // + // ```python + // class Foo[**P, T]: ... + // + // Foo[Any, int] # P: (Any, /), T: int + // ``` + Type::Dynamic(DynamicType::Any) => { + return Ok(Type::paramspec_value_callable( + db, + Parameters::gradual_form(), + )); + } + + _ => {} } } - _ => { - if let Some(builder) = self.context.report_lint(&INVALID_PARAMSPEC, default) { - builder.into_diagnostic( - "The default value to `ParamSpec` must be either a list of types, \ - `ParamSpec`, or `...`", - ); - } - Type::unknown() - } + + _ => {} } + + if let Some(builder) = self.context.report_lint(&INVALID_TYPE_ARGUMENTS, expr) { + builder.into_diagnostic( + "Type argument for `ParamSpec` must be either \ + a list of types, `ParamSpec`, `Concatenate`, or `...`", + ); + } + + Err(()) } fn infer_typevartuple_definition( @@ -9121,10 +9321,23 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let ast::ExprAttribute { value, attr, .. } = attribute; - let value_type = self.infer_maybe_standalone_expression(value, TypeContext::default()); + let mut value_type = self.infer_maybe_standalone_expression(value, TypeContext::default()); let db = self.db(); let mut constraint_keys = vec![]; + if let Type::KnownInstance(KnownInstanceType::TypeVar(typevar)) = value_type + && typevar.is_paramspec(db) + && let Some(bound_typevar) = bind_typevar( + db, + self.index, + self.scope().file_scope_id(db), + self.typevar_binding_context, + typevar, + ) + { + value_type = Type::TypeVar(bound_typevar); + } + let mut assigned_type = None; if let Some(place_expr) = PlaceExpr::try_from_expr(attribute) { let (resolved, keys) = self.infer_place_load( @@ -11176,7 +11389,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { std::slice::from_ref(&*subscript.slice) }; - // TODO: Remove this once we support ParamSpec and Concatenate properly. This is necessary + // TODO: Remove this once we support Concatenate properly. This is necessary // to avoid a lot of false positives downstream, because we can't represent the typevar- // specialized `Callable` types yet. let num_arguments = arguments.len(); @@ -11184,27 +11397,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let first_arg = &arguments[0]; let second_arg = &arguments[1]; - if first_arg.is_name_expr() { - let first_arg_ty = self.infer_expression(first_arg, TypeContext::default()); - if let Type::KnownInstance(KnownInstanceType::TypeVar(typevar)) = - first_arg_ty - && typevar.kind(self.db()).is_paramspec() - { - return todo_type!("Callable[..] specialized with ParamSpec"); - } - - if let Some(builder) = - self.context.report_lint(&INVALID_TYPE_FORM, subscript) - { - builder.into_diagnostic(format_args!( - "The first argument to `Callable` must be either a list of types, \ - ParamSpec, Concatenate, or `...`", - )); - } - return Type::KnownInstance(KnownInstanceType::Callable( - CallableType::unknown(self.db()), - )); - } else if first_arg.is_subscript_expr() { + if first_arg.is_subscript_expr() { let first_arg_ty = self.infer_expression(first_arg, TypeContext::default()); if let Type::Dynamic(DynamicType::UnknownGeneric(generic_context)) = first_arg_ty @@ -11436,22 +11629,18 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let db = self.db(); let slice_node = subscript.slice.as_ref(); - // Extract type arguments from the subscript - let type_arguments: Vec> = match slice_node { + let exactly_one_paramspec = generic_context.exactly_one_paramspec(db); + let (type_arguments, store_inferred_type_arguments) = match slice_node { ast::Expr::Tuple(tuple) => { - let types: Vec<_> = tuple - .elts - .iter() - .map(|elt| self.infer_type_expression(elt)) - .collect(); - self.store_expression_type( - slice_node, - Type::heterogeneous_tuple(db, types.iter().copied()), - ); - types + if exactly_one_paramspec { + (std::slice::from_ref(slice_node), false) + } else { + (tuple.elts.as_slice(), true) + } } - _ => vec![self.infer_type_expression(slice_node)], + _ => (std::slice::from_ref(slice_node), false), }; + let mut inferred_type_arguments = Vec::with_capacity(type_arguments.len()); let typevars = generic_context.variables(db); let typevars_len = typevars.len(); @@ -11464,7 +11653,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // Helper to get the AST node corresponding to the type argument at `index`. let get_node = |index: usize| -> ast::AnyNodeRef<'_> { match slice_node { - ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => elts + ast::Expr::Tuple(ast::ExprTuple { elts, .. }) if !exactly_one_paramspec => elts .get(index) .expect("type argument index should not be out of range") .into(), @@ -11476,10 +11665,28 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { for (index, item) in typevars.zip_longest(type_arguments.iter()).enumerate() { match item { - EitherOrBoth::Both(typevar, &provided_type) => { + EitherOrBoth::Both(typevar, expr) => { if typevar.default_type(db).is_some() { typevar_with_defaults += 1; } + + let provided_type = if typevar.is_paramspec(db) { + match self.infer_paramspec_explicit_specialization_value( + expr, + exactly_one_paramspec, + ) { + Ok(paramspec_value) => paramspec_value, + Err(()) => { + has_error = true; + Type::unknown() + } + } + } else { + self.infer_type_expression(expr) + }; + + inferred_type_arguments.push(provided_type); + // TODO consider just accepting the given specialization without checking // against bounds/constraints, but recording the expression for deferred // checking at end of scope. This would avoid a lot of cycles caused by eagerly @@ -11543,17 +11750,20 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } None => {} } + specialization_types.push(Some(provided_type)); } EitherOrBoth::Left(typevar) => { if typevar.default_type(db).is_none() { + // This is an error case, so no need to push into the specialization types. missing_typevars.push(typevar); } else { typevar_with_defaults += 1; + specialization_types.push(None); } - specialization_types.push(None); } - EitherOrBoth::Right(_) => { + EitherOrBoth::Right(expr) => { + inferred_type_arguments.push(self.infer_type_expression(expr)); first_excess_type_argument_index.get_or_insert(index); } } @@ -11605,10 +11815,23 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { has_error = true; } + if store_inferred_type_arguments { + self.store_expression_type( + slice_node, + Type::heterogeneous_tuple(db, inferred_type_arguments), + ); + } + if has_error { let unknowns = generic_context .variables(self.db()) - .map(|_| Some(Type::unknown())) + .map(|typevar| { + Some(if typevar.is_paramspec(db) { + Type::paramspec_value_callable(db, Parameters::unknown()) + } else { + Type::unknown() + }) + }) .collect::>(); return specialize(&unknowns); } @@ -12044,10 +12267,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Type::Dynamic( DynamicType::TodoUnpack | DynamicType::TodoStarredExpression, ) => true, - Type::NominalInstance(nominal) => matches!( - nominal.known_class(self.db()), - Some(KnownClass::TypeVarTuple | KnownClass::ParamSpec) - ), + Type::NominalInstance(nominal) => { + nominal.has_known_class(self.db(), KnownClass::TypeVarTuple) + } _ => false, }, true, diff --git a/crates/ty_python_semantic/src/types/infer/builder/annotation_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/annotation_expression.rs index e143b5c7fa..ce3b2a4a92 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/annotation_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/annotation_expression.rs @@ -190,12 +190,16 @@ impl<'db> TypeInferenceBuilder<'db, '_> { } ast::Expr::Attribute(attribute) => match attribute.ctx { - ast::ExprContext::Load => infer_name_or_attribute( - self.infer_attribute_expression(attribute), - annotation, - self, - pep_613_policy, - ), + ast::ExprContext::Load => { + let attribute_type = self.infer_attribute_expression(attribute); + if let Type::TypeVar(typevar) = attribute_type + && typevar.paramspec_attr(self.db()).is_some() + { + TypeAndQualifiers::declared(attribute_type) + } else { + infer_name_or_attribute(attribute_type, annotation, self, pep_613_policy) + } + } ast::ExprContext::Invalid => TypeAndQualifiers::declared(Type::unknown()), ast::ExprContext::Store | ast::ExprContext::Del => TypeAndQualifiers::declared( todo_type!("Attribute expression annotation in Store/Del context"), diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index 56b0db5a09..114cedb734 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -3,19 +3,21 @@ use ruff_python_ast as ast; use super::{DeferredExpressionState, TypeInferenceBuilder}; use crate::FxOrderSet; +use crate::semantic_index::semantic_index; use crate::types::diagnostic::{ self, INVALID_TYPE_FORM, NON_SUBSCRIPTABLE, report_invalid_argument_number_to_special_form, report_invalid_arguments_to_callable, }; +use crate::types::generics::bind_typevar; use crate::types::infer::builder::InnerExpressionInferenceState; -use crate::types::signatures::{Parameter, Parameters, Signature}; +use crate::types::signatures::Signature; use crate::types::string_annotation::parse_string_annotation; use crate::types::tuple::{TupleSpecBuilder, TupleType}; -use crate::types::visitor::any_over_type; use crate::types::{ BindingContext, CallableType, DynamicType, GenericContext, IntersectionBuilder, KnownClass, - KnownInstanceType, LintDiagnosticGuard, SpecialFormType, SubclassOfType, Type, TypeAliasType, - TypeContext, TypeIsType, TypeMapping, UnionBuilder, UnionType, todo_type, + KnownInstanceType, LintDiagnosticGuard, Parameter, Parameters, SpecialFormType, SubclassOfType, + Type, TypeAliasType, TypeContext, TypeIsType, TypeMapping, UnionBuilder, UnionType, + any_over_type, todo_type, }; /// Type expressions @@ -821,7 +823,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { ) } - fn infer_subscript_type_expression( + pub(super) fn infer_subscript_type_expression( &mut self, subscript: &ast::ExprSubscript, value_ty: Type<'db>, @@ -1749,21 +1751,22 @@ impl<'db> TypeInferenceBuilder<'db, '_> { // `Callable[]`. return None; } - if any_over_type( - self.db(), - self.infer_name_load(name), - &|ty| match ty { - Type::KnownInstance(known_instance) => { - known_instance.class(self.db()) == KnownClass::ParamSpec - } - Type::NominalInstance(nominal) => { - nominal.has_known_class(self.db(), KnownClass::ParamSpec) - } - _ => false, - }, - true, - ) { - return Some(Parameters::todo()); + let name_ty = self.infer_name_load(name); + if let Type::KnownInstance(KnownInstanceType::TypeVar(typevar)) = name_ty + && typevar.is_paramspec(self.db()) + { + let index = semantic_index(self.db(), self.scope().file(self.db())); + let Some(bound_typevar) = bind_typevar( + self.db(), + index, + self.scope().file_scope_id(self.db()), + self.typevar_binding_context, + typevar, + ) else { + // TODO: What to do here? + return None; + }; + return Some(Parameters::paramspec(self.db(), bound_typevar)); } } _ => {} diff --git a/crates/ty_python_semantic/src/types/protocol_class.rs b/crates/ty_python_semantic/src/types/protocol_class.rs index 80ce2af47c..5d3719b8d9 100644 --- a/crates/ty_python_semantic/src/types/protocol_class.rs +++ b/crates/ty_python_semantic/src/types/protocol_class.rs @@ -6,7 +6,7 @@ use itertools::Itertools; use ruff_python_ast::name::Name; use rustc_hash::FxHashMap; -use crate::types::TypeContext; +use crate::types::{CallableTypeKind, TypeContext}; use crate::{ Db, FxOrderSet, place::{Definedness, Place, PlaceAndQualifiers, place_from_bindings, place_from_declarations}, @@ -986,5 +986,9 @@ fn protocol_bind_self<'db>( callable: CallableType<'db>, self_type: Option>, ) -> CallableType<'db> { - CallableType::new(db, callable.signatures(db).bind_self(db, self_type), false) + CallableType::new( + db, + callable.signatures(db).bind_self(db, self_type), + CallableTypeKind::Regular, + ) } diff --git a/crates/ty_python_semantic/src/types/signatures.rs b/crates/ty_python_semantic/src/types/signatures.rs index c76a086cfa..ffc224c8d5 100644 --- a/crates/ty_python_semantic/src/types/signatures.rs +++ b/crates/ty_python_semantic/src/types/signatures.rs @@ -29,10 +29,10 @@ use crate::types::generics::{ }; use crate::types::infer::nearest_enclosing_class; use crate::types::{ - ApplyTypeMappingVisitor, BindingContext, BoundTypeVarInstance, ClassLiteral, + ApplyTypeMappingVisitor, BindingContext, BoundTypeVarInstance, CallableTypeKind, ClassLiteral, FindLegacyTypeVarsVisitor, HasRelationToVisitor, IsDisjointVisitor, IsEquivalentVisitor, - KnownClass, MaterializationKind, NormalizedVisitor, TypeContext, TypeMapping, TypeRelation, - VarianceInferable, todo_type, + KnownClass, MaterializationKind, NormalizedVisitor, ParamSpecAttrKind, TypeContext, + TypeMapping, TypeRelation, VarianceInferable, todo_type, }; use crate::{Db, FxOrderSet}; use ruff_python_ast::{self as ast, name::Name}; @@ -151,6 +151,10 @@ impl<'db> CallableSignature<'db> { self.overloads.iter() } + pub(crate) fn as_slice(&self) -> &[Signature<'db>] { + &self.overloads + } + pub(crate) fn with_inherited_generic_context( &self, db: &'db dyn Db, @@ -197,6 +201,122 @@ impl<'db> CallableSignature<'db> { tcx: TypeContext<'db>, visitor: &ApplyTypeMappingVisitor<'db>, ) -> Self { + fn try_apply_type_mapping_for_paramspec<'db>( + db: &'db dyn Db, + self_signature: &Signature<'db>, + prefix_parameters: &[Parameter<'db>], + paramspec_value: Type<'db>, + type_mapping: &TypeMapping<'_, 'db>, + tcx: TypeContext<'db>, + visitor: &ApplyTypeMappingVisitor<'db>, + ) -> Option> { + match paramspec_value { + Type::TypeVar(typevar) if typevar.is_paramspec(db) => { + Some(CallableSignature::single(Signature { + generic_context: self_signature.generic_context.map(|context| { + type_mapping.update_signature_generic_context(db, context) + }), + definition: self_signature.definition, + parameters: Parameters::new( + db, + prefix_parameters + .iter() + .map(|param| { + param.apply_type_mapping_impl(db, type_mapping, tcx, visitor) + }) + .chain([ + Parameter::variadic(Name::new_static("args")) + .with_annotated_type(Type::TypeVar( + typevar + .with_paramspec_attr(db, ParamSpecAttrKind::Args), + )), + Parameter::keyword_variadic(Name::new_static("kwargs")) + .with_annotated_type(Type::TypeVar( + typevar + .with_paramspec_attr(db, ParamSpecAttrKind::Kwargs), + )), + ]), + ), + return_ty: self_signature + .return_ty + .map(|ty| ty.apply_type_mapping_impl(db, type_mapping, tcx, visitor)), + })) + } + Type::Callable(callable) + if matches!(callable.kind(db), CallableTypeKind::ParamSpecValue) => + { + Some(CallableSignature::from_overloads( + callable.signatures(db).iter().map(|signature| Signature { + generic_context: self_signature.generic_context.map(|context| { + type_mapping.update_signature_generic_context(db, context) + }), + definition: signature.definition, + parameters: Parameters::new( + db, + prefix_parameters + .iter() + .map(|param| { + param.apply_type_mapping_impl( + db, + type_mapping, + tcx, + visitor, + ) + }) + .chain(signature.parameters().iter().cloned()), + ), + return_ty: self_signature.return_ty.map(|ty| { + ty.apply_type_mapping_impl(db, type_mapping, tcx, visitor) + }), + }), + )) + } + _ => None, + } + } + + match type_mapping { + TypeMapping::Specialization(specialization) => { + if let [self_signature] = self.overloads.as_slice() + && let Some((prefix_parameters, paramspec)) = self_signature + .parameters + .find_paramspec_from_args_kwargs(db) + && let Some(paramspec_value) = specialization.get(db, paramspec) + && let Some(result) = try_apply_type_mapping_for_paramspec( + db, + self_signature, + prefix_parameters, + paramspec_value, + type_mapping, + tcx, + visitor, + ) + { + return result; + } + } + TypeMapping::PartialSpecialization(partial) => { + if let [self_signature] = self.overloads.as_slice() + && let Some((prefix_parameters, paramspec)) = self_signature + .parameters + .find_paramspec_from_args_kwargs(db) + && let Some(paramspec_value) = partial.get(db, paramspec) + && let Some(result) = try_apply_type_mapping_for_paramspec( + db, + self_signature, + prefix_parameters, + paramspec_value, + type_mapping, + tcx, + visitor, + ) + { + return result; + } + } + _ => {} + } + Self::from_overloads( self.overloads .iter() @@ -1321,15 +1441,18 @@ impl<'db> VarianceInferable<'db> for &Signature<'db> { } } -#[derive(Clone, Debug, PartialEq, Eq, Hash, salsa::Update, get_size2::GetSize)] -pub(crate) struct Parameters<'db> { - // TODO: use SmallVec here once invariance bug is fixed - value: Vec>, +// TODO: the spec also allows signatures like `Concatenate[int, ...]` or `Concatenate[int, P]`, +// which have some number of required positional-only parameters followed by a gradual form or a +// `ParamSpec`. Our representation will need some adjustments to represent that. - /// Whether this parameter list represents a gradual form using `...` as the only parameter. - /// - /// If this is `true`, the `value` will still contain the variadic and keyword-variadic - /// parameters. +/// The kind of parameter list represented. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, Hash, salsa::Update, get_size2::GetSize)] +pub(crate) enum ParametersKind<'db> { + /// A standard parameter list. + #[default] + Standard, + + /// Represents a gradual parameter list using `...` as the only parameter. /// /// Per [the typing specification], any signature with a variadic and a keyword-variadic /// argument, both annotated (explicitly or implicitly) as `Any` or `Unknown`, is considered @@ -1340,35 +1463,68 @@ pub(crate) struct Parameters<'db> { /// /// Note: This flag can also result from invalid forms of `Callable` annotations. /// - /// TODO: the spec also allows signatures like `Concatenate[int, ...]`, which have some number - /// of required positional parameters followed by a gradual form. Our representation will need - /// some adjustments to represent that. + /// [the typing specification]: https://typing.python.org/en/latest/spec/callables.html#meaning-of-in-callable + Gradual, + + /// Represents a parameter list containing a `ParamSpec` as the only parameter. /// - /// [the typing specification]: https://typing.python.org/en/latest/spec/callables.html#meaning-of-in-callable - is_gradual: bool, + /// Note that this is distinct from a parameter list _containing_ a `ParamSpec` which is + /// considered a standard parameter list that just contains a `ParamSpec`. + // TODO: Maybe we should use `find_paramspec_from_args_kwargs` instead of storing the typevar + // here? + ParamSpec(BoundTypeVarInstance<'db>), +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, salsa::Update, get_size2::GetSize)] +pub(crate) struct Parameters<'db> { + // TODO: use SmallVec here once invariance bug is fixed + value: Vec>, + kind: ParametersKind<'db>, } impl<'db> Parameters<'db> { + /// Create a new parameter list from an iterator of parameters. + /// + /// The kind of the parameter list is determined based on the provided parameters. + /// Specifically, if the parameters is made up of `*args` and `**kwargs` only, it checks + /// their annotated types to determine if they represent a gradual form or a `ParamSpec`. pub(crate) fn new( - _db: &'db dyn Db, + db: &'db dyn Db, parameters: impl IntoIterator>, ) -> Self { let value: Vec> = parameters.into_iter().collect(); - let is_gradual = value.len() == 2 - && value - .iter() - .any(|p| p.is_variadic() && p.annotated_type().is_none_or(|ty| ty.is_dynamic())) - && value.iter().any(|p| { - p.is_keyword_variadic() && p.annotated_type().is_none_or(|ty| ty.is_dynamic()) - }); - Self { value, is_gradual } + let mut kind = ParametersKind::Standard; + if let [p1, p2] = value.as_slice() + && p1.is_variadic() + && p2.is_keyword_variadic() + { + match (p1.annotated_type(), p2.annotated_type()) { + (None | Some(Type::Dynamic(_)), None | Some(Type::Dynamic(_))) => { + kind = ParametersKind::Gradual; + } + (Some(Type::TypeVar(args_typevar)), Some(Type::TypeVar(kwargs_typevar))) => { + if let (Some(ParamSpecAttrKind::Args), Some(ParamSpecAttrKind::Kwargs)) = ( + args_typevar.paramspec_attr(db), + kwargs_typevar.paramspec_attr(db), + ) { + let typevar = args_typevar.without_paramspec_attr(db); + if typevar.is_same_typevar_as(db, kwargs_typevar.without_paramspec_attr(db)) + { + kind = ParametersKind::ParamSpec(typevar); + } + } + } + _ => {} + } + } + Self { value, kind } } /// Create an empty parameter list. pub(crate) fn empty() -> Self { Self { value: Vec::new(), - is_gradual: false, + kind: ParametersKind::Standard, } } @@ -1376,8 +1532,12 @@ impl<'db> Parameters<'db> { self.value.as_slice() } + pub(crate) const fn kind(&self) -> ParametersKind<'db> { + self.kind + } + pub(crate) const fn is_gradual(&self) -> bool { - self.is_gradual + matches!(self.kind, ParametersKind::Gradual) } /// Return todo parameters: (*args: Todo, **kwargs: Todo) @@ -1389,7 +1549,7 @@ impl<'db> Parameters<'db> { Parameter::keyword_variadic(Name::new_static("kwargs")) .with_annotated_type(todo_type!("todo signature **kwargs")), ], - is_gradual: true, + kind: ParametersKind::Gradual, } } @@ -1406,7 +1566,21 @@ impl<'db> Parameters<'db> { Parameter::keyword_variadic(Name::new_static("kwargs")) .with_annotated_type(Type::Dynamic(DynamicType::Any)), ], - is_gradual: true, + kind: ParametersKind::Gradual, + } + } + + pub(crate) fn paramspec(db: &'db dyn Db, typevar: BoundTypeVarInstance<'db>) -> Self { + Self { + value: vec![ + Parameter::variadic(Name::new_static("args")).with_annotated_type(Type::TypeVar( + typevar.with_paramspec_attr(db, ParamSpecAttrKind::Args), + )), + Parameter::keyword_variadic(Name::new_static("kwargs")).with_annotated_type( + Type::TypeVar(typevar.with_paramspec_attr(db, ParamSpecAttrKind::Kwargs)), + ), + ], + kind: ParametersKind::ParamSpec(typevar), } } @@ -1424,7 +1598,7 @@ impl<'db> Parameters<'db> { Parameter::keyword_variadic(Name::new_static("kwargs")) .with_annotated_type(Type::Dynamic(DynamicType::Unknown)), ], - is_gradual: true, + kind: ParametersKind::Gradual, } } @@ -1436,10 +1610,48 @@ impl<'db> Parameters<'db> { Parameter::keyword_variadic(Name::new_static("kwargs")) .with_annotated_type(Type::object()), ], - is_gradual: false, + kind: ParametersKind::Standard, } } + /// Returns the bound `ParamSpec` type variable if the parameters contain a `ParamSpec`. + pub(crate) fn find_paramspec_from_args_kwargs<'a>( + &'a self, + db: &'db dyn Db, + ) -> Option<(&'a [Parameter<'db>], BoundTypeVarInstance<'db>)> { + let [prefix @ .., maybe_args, maybe_kwargs] = self.value.as_slice() else { + return None; + }; + + if !maybe_args.is_variadic() || !maybe_kwargs.is_keyword_variadic() { + return None; + } + + let (Type::TypeVar(args_typevar), Type::TypeVar(kwargs_typevar)) = + (maybe_args.annotated_type()?, maybe_kwargs.annotated_type()?) + else { + return None; + }; + + if matches!( + ( + args_typevar.paramspec_attr(db), + kwargs_typevar.paramspec_attr(db) + ), + ( + Some(ParamSpecAttrKind::Args), + Some(ParamSpecAttrKind::Kwargs) + ) + ) { + let typevar = args_typevar.without_paramspec_attr(db); + if typevar.is_same_typevar_as(db, kwargs_typevar.without_paramspec_attr(db)) { + return Some((prefix, typevar)); + } + } + + None + } + fn from_parameters( db: &'db dyn Db, definition: Definition<'db>, @@ -1627,13 +1839,13 @@ impl<'db> Parameters<'db> { // Note that we've already flipped the materialization in Signature.apply_type_mapping_impl(), // so the "top" materialization here is the bottom materialization of the whole Signature. // It might make sense to flip the materialization here instead. - TypeMapping::Materialize(MaterializationKind::Top) if self.is_gradual => { + TypeMapping::Materialize(MaterializationKind::Top) if self.is_gradual() => { Parameters::object() } // TODO: This is wrong, the empty Parameters is not a subtype of all materializations. // The bottom materialization is not currently representable and implementing it // properly requires extending the Parameters struct. - TypeMapping::Materialize(MaterializationKind::Bottom) if self.is_gradual => { + TypeMapping::Materialize(MaterializationKind::Bottom) if self.is_gradual() => { Parameters::empty() } _ => Self { @@ -1642,7 +1854,7 @@ impl<'db> Parameters<'db> { .iter() .map(|param| param.apply_type_mapping_impl(db, type_mapping, tcx, visitor)) .collect(), - is_gradual: self.is_gradual, + kind: self.kind, }, } }