Python/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff synced 2026-01-08 15:14:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

[ty] implement typing.NewType by adding Type::NewTypeInstance

Browse Source
This commit is contained in:
Jack O'Connor
2025-10-23 10:10:10 -07:00
parent 039a69fa8c
commit 5f3e086ee4
25 changed files with 1343 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

382
crates/ty_python_semantic/resources/mdtest/annotations/new_types.md
View File

@@ -1,7 +1,5 @@
# NewType
Currently, ty doesn't support `typing.NewType` in type annotations.
## Valid forms
```py
@@ -12,13 +10,389 @@ X = GenericAlias(type, ())
A = NewType("A", int)
# TODO: typeshed for `typing.GenericAlias` uses `type` for the first argument. `NewType` should be special-cased
# to be compatible with `type`
# error: [invalid-argument-type] "Argument to function `__new__` is incorrect: Expected `type`, found `NewType`"
# error: [invalid-argument-type] "Argument to function `__new__` is incorrect: Expected `type`, found `<NewType pseudo-class 'A'>`"
B = GenericAlias(A, ())
def _(
a: A,
b: B,
):
reveal_type(a) # revealed: @Todo(Support for `typing.NewType` instances in type expressions)
reveal_type(a) # revealed: A
reveal_type(b) # revealed: @Todo(Support for `typing.GenericAlias` instances in type expressions)
```
## Subtyping
The basic purpose of `NewType` is that it acts like a subtype of its base, but not the exact same
type (i.e. not an alias).
```py
from typing_extensions import NewType
from ty_extensions import static_assert, is_subtype_of, is_equivalent_to
Foo = NewType("Foo", int)
Bar = NewType("Bar", Foo)
static_assert(is_subtype_of(Foo, int))
static_assert(not is_equivalent_to(Foo, int))
static_assert(is_subtype_of(Bar, Foo))
static_assert(is_subtype_of(Bar, int))
static_assert(not is_equivalent_to(Bar, Foo))
Foo(42)
Foo(Foo(42)) # allowed: `Foo` is a subtype of `int`.
Foo(Bar(Foo(42))) # allowed: `Bar` is a subtype of `int`.
Foo(True) # allowed: `bool` is a subtype of `int`.
Foo("forty-two") # error: [invalid-argument-type] "Argument is incorrect: Expected `int`, found `Literal["forty-two"]`"
def f(_: int): ...
def g(_: Foo): ...
def h(_: Bar): ...
f(42)
f(Foo(42))
f(Bar(Foo(42)))
g(42) # error: [invalid-argument-type] "Argument to function `g` is incorrect: Expected `Foo`, found `Literal[42]`"
g(Foo(42))
g(Bar(Foo(42)))
h(42) # error: [invalid-argument-type] "Argument to function `h` is incorrect: Expected `Bar`, found `Literal[42]`"
h(Foo(42)) # error: [invalid-argument-type] "Argument to function `h` is incorrect: Expected `Bar`, found `Foo`"
h(Bar(Foo(42)))
```
## Member and method lookup work
```py
from typing_extensions import NewType
class Foo:
foo_member: str = "hello"
def foo_method(self) -> int:
return 42
Bar = NewType("Bar", Foo)
Baz = NewType("Baz", Bar)
baz = Baz(Bar(Foo()))
reveal_type(baz.foo_member) # revealed: str
reveal_type(baz.foo_method()) # revealed: int
```
We also infer member access on the `NewType` pseudo-type itself correctly:
```py
reveal_type(Bar.__supertype__) # revealed: type | NewType
reveal_type(Baz.__supertype__) # revealed: type | NewType
```
## `NewType` wrapper functions are `Callable`
```py
from collections.abc import Callable
from typing_extensions import NewType
from ty_extensions import CallableTypeOf
Foo = NewType("Foo", int)
def _(obj: CallableTypeOf[Foo]):
reveal_type(obj) # revealed: (int, /) -> Foo
def f(_: Callable[[int], Foo]): ...
f(Foo)
map(Foo, [1, 2, 3])
def g(_: Callable[[str], Foo]): ...
g(Foo) # error: [invalid-argument-type]
```
## `NewType` instances are `Callable` if the base type is
```py
from typing import NewType, Callable, Any
from ty_extensions import CallableTypeOf
N = NewType("N", int)
i = N(42)
y: Callable[..., Any] = i # error: [invalid-assignment] "Object of type `N` is not assignable to `(...) -> Any`"
# error: [invalid-type-form] "Expected the first argument to `ty_extensions.CallableTypeOf` to be a callable object, but got an object of type `N`"
def f(x: CallableTypeOf[i]):
reveal_type(x) # revealed: Unknown
class SomethingCallable:
def __call__(self, a: str) -> bytes:
raise NotImplementedError
N2 = NewType("N2", SomethingCallable)
j = N2(SomethingCallable())
z: Callable[[str], bytes] = j # fine
def g(x: CallableTypeOf[j]):
reveal_type(x) # revealed: (a: str) -> bytes
```
## The name must be a string literal
```py
from typing_extensions import NewType
def _(name: str) -> None:
_ = NewType(name, int) # error: [invalid-newtype] "The first argument to `NewType` must be a string literal"
```
However, the literal doesn't necessarily need to be inline, as long as we infer it:
```py
name = "Foo"
Foo = NewType(name, int)
reveal_type(Foo) # revealed: <NewType pseudo-class 'Foo'>
```
## The second argument must be a class type or another newtype
Other typing constructs like `Union` are not allowed.
```py
from typing_extensions import NewType
# error: [invalid-newtype] "invalid base for `typing.NewType`"
Foo = NewType("Foo", int | str)
```
We don't emit the "invalid base" diagnostic for `Unknown`, because that typically results from other
errors that already have a diagnostic, and there's no need to pile on. For example, this mistake
gives you an "Int literals are not allowed" error, and we'd rather not see an "invalid base" error
on top of that:
```py
# error: [invalid-type-form] "Int literals are not allowed in this context in a type expression"
Foo = NewType("Foo", 42)
```
## A `NewType` definition must be a simple variable assignment
```py
from typing import NewType
N: NewType = NewType("N", int) # error: [invalid-newtype] "A `NewType` definition must be a simple variable assignment"
```
## Newtypes can be cyclic in various ways
Cyclic newtypes are kind of silly, but it's possible for the user to express them, and it's
important that we don't go into infinite recursive loops and crash with a stack overflow. In fact,
this is *why* base type evaluation is deferred; otherwise Salsa itself would crash.
```py
from typing_extensions import NewType, reveal_type, cast
# Define a directly cyclic newtype.
A = NewType("A", "A")
reveal_type(A) # revealed: <NewType pseudo-class 'A'>
# Typechecking still works. We can't construct an `A` "honestly", but we can `cast` into one.
a: A
a = 42 # error: [invalid-assignment] "Object of type `Literal[42]` is not assignable to `A`"
a = A(42) # error: [invalid-argument-type] "Argument is incorrect: Expected `A`, found `Literal[42]`"
a = cast(A, 42)
reveal_type(a) # revealed: A
# A newtype cycle might involve more than one step.
B = NewType("B", "C")
C = NewType("C", "B")
reveal_type(B) # revealed: <NewType pseudo-class 'B'>
reveal_type(C) # revealed: <NewType pseudo-class 'C'>
b: B = cast(B, 42)
c: C = C(b)
reveal_type(b) # revealed: B
reveal_type(c) # revealed: C
# Cyclic types behave in surprising ways. These assignments are legal, even though B and C aren't
# the same type, because each of them is a subtype of the other.
b = c
c = b
# Another newtype could inherit from a cyclic one.
D = NewType("D", C)
reveal_type(D) # revealed: <NewType pseudo-class 'D'>
d: D
d = D(42) # error: [invalid-argument-type] "Argument is incorrect: Expected `C`, found `Literal[42]`"
d = D(c)
d = D(b) # Allowed, the same surprise as above. B and C are subtypes of each other.
reveal_type(d) # revealed: D
```
Normal classes can't inherit from newtypes, but generic classes can be parametrized with them, so we
also need to detect "ordinary" type cycles that happen to involve a newtype.
```py
E = NewType("E", list["E"])
reveal_type(E) # revealed: <NewType pseudo-class 'E'>
e: E = E([])
reveal_type(e) # revealed: E
reveal_type(E(E(E(E(E([])))))) # revealed: E
reveal_type(E([E([E([]), E([E([])])]), E([])])) # revealed: E
E(["foo"]) # error: [invalid-argument-type]
E(E(E(["foo"]))) # error: [invalid-argument-type]
```
## `NewType` wrapping preserves singleton-ness and single-valued-ness
```py
from typing_extensions import NewType
from ty_extensions import is_singleton, is_single_valued, static_assert
from types import EllipsisType
A = NewType("A", EllipsisType)
static_assert(is_singleton(A))
static_assert(is_single_valued(A))
reveal_type(type(A(...)) is EllipsisType) # revealed: Literal[True]
# TODO: This should be `Literal[True]` also.
reveal_type(A(...) is ...) # revealed: bool
B = NewType("B", int)
static_assert(not is_singleton(B))
static_assert(not is_single_valued(B))
```
## `NewType`s of tuples can be iterated/unpacked
```py
from typing import NewType
N = NewType("N", tuple[int, str])
a, b = N((1, "foo"))
reveal_type(a) # revealed: int
reveal_type(b) # revealed: str
```
## `isinstance` of a `NewType` instance and its base class is inferred as `Literal[True]`
```py
from typing import NewType
N = NewType("N", int)
def f(x: N):
reveal_type(isinstance(x, int)) # revealed: Literal[True]
```
However, a `NewType` isn't a real class, so it isn't a valid second argument to `isinstance`:
```py
def f(x: N):
# error: [invalid-argument-type] "Argument to function `isinstance` is incorrect"
reveal_type(isinstance(x, N)) # revealed: bool
```
Because of that, we don't generate any narrowing constraints for it:
```py
def f(x: N | str):
if isinstance(x, N): # error: [invalid-argument-type]
reveal_type(x) # revealed: N | str
else:
reveal_type(x) # revealed: N | str
```
## Trying to subclass a `NewType` produces an error matching CPython
<!-- snapshot-diagnostics -->
```py
from typing import NewType
X = NewType("X", int)
class Foo(X): ... # error: [invalid-base]
```
## Don't narrow `NewType`-wrapped `Enum`s inside of match arms
`Literal[Foo.X]` is actually disjoint from `N` here:
```py
from enum import Enum
from typing import NewType
class Foo(Enum):
X = 0
Y = 1
N = NewType("N", Foo)
def f(x: N):
match x:
case Foo.X:
reveal_type(x) # revealed: N
case Foo.Y:
reveal_type(x) # revealed: N
case _:
reveal_type(x) # revealed: N
```
## We don't support `NewType` on Python 3.9
We implement `typing.NewType` as a `KnownClass`, but in Python 3.9 it's actually a function, so all
we get is the `Any` annotations from typeshed. However, `typing_extensions.NewType` is always a
class. This could be improved in the future, but Python 3.9 is now end-of-life, so it's not
high-priority.
```toml
[environment]
python-version = "3.9"
```
```py
from typing import NewType
Foo = NewType("Foo", int)
reveal_type(Foo) # revealed: Any
reveal_type(Foo(42)) # revealed: Any
from typing_extensions import NewType
Bar = NewType("Bar", int)
reveal_type(Bar) # revealed: <NewType pseudo-class 'Bar'>
reveal_type(Bar(42)) # revealed: Bar
```
## The base of a `NewType` can't be a protocol class or a `TypedDict`
<!-- snapshot-diagnostics -->
```py
from typing import NewType, Protocol, TypedDict
class Id(Protocol):
code: int
UserId = NewType("UserId", Id) # error: [invalid-newtype]
class Foo(TypedDict):
a: int
Bar = NewType("Bar", Foo) # error: [invalid-newtype]
```
## TODO: A `NewType` cannot be generic
```py
from typing import Any, NewType, TypeVar
# All of these are allowed.
A = NewType("A", list)
B = NewType("B", list[int])
B = NewType("B", list[Any])
# But a free typevar is not allowed.
T = TypeVar("T")
C = NewType("C", list[T]) # TODO: should be "error: [invalid-newtype]"
```

6
crates/ty_python_semantic/resources/mdtest/class/super.md
View File

@@ -66,7 +66,7 @@ synthesized `Protocol`s that cannot be upcast to, or interpreted as, a non-`obje
```py
import types
from typing_extensions import Callable, TypeIs, Literal, TypedDict
from typing_extensions import Callable, TypeIs, Literal, NewType, TypedDict
def f(): ...
@@ -81,6 +81,8 @@ class SomeTypedDict(TypedDict):
x: int
y: bytes
N = NewType("N", int)
# revealed: <super: <class 'object'>, FunctionType>
reveal_type(super(object, f))
# revealed: <super: <class 'object'>, WrapperDescriptorType>
@@ -95,6 +97,8 @@ reveal_type(super(object, Alias))
reveal_type(super(object, Foo().method))
# revealed: <super: <class 'object'>, property>
reveal_type(super(object, Foo.some_property))
# revealed: <super: <class 'object'>, int>
reveal_type(super(object, N(42)))
def g(x: object) -> TypeIs[list[object]]:
return isinstance(x, list)

58
crates/ty_python_semantic/resources/mdtest/snapshots/new_types.md_-_NewType_-_The_base_of_a_`NewTy…_(9847ea9eddc316b4).snap Normal file
View File

@@ -0,0 +1,58 @@
---
source: crates/ty_test/src/lib.rs
expression: snapshot
---
---
mdtest name: new_types.md - NewType - The base of a `NewType` can't be a protocol class or a `TypedDict`
mdtest path: crates/ty_python_semantic/resources/mdtest/annotations/new_types.md
---
# Python source files
## mdtest_snippet.py
```
1 | from typing import NewType, Protocol, TypedDict
2 |
3 | class Id(Protocol):
4 | code: int
5 |
6 | UserId = NewType("UserId", Id) # error: [invalid-newtype]
7 |
8 | class Foo(TypedDict):
9 | a: int
10 |
11 | Bar = NewType("Bar", Foo) # error: [invalid-newtype]
```
# Diagnostics
```
error[invalid-newtype]: invalid base for `typing.NewType`
--> src/mdtest_snippet.py:6:28
|
4 | code: int
5 |
6 | UserId = NewType("UserId", Id) # error: [invalid-newtype]
| ^^ type `Id`
7 |
8 | class Foo(TypedDict):
|
info: The base of a `NewType` is not allowed to be a protocol class.
info: rule `invalid-newtype` is enabled by default
```
```
error[invalid-newtype]: invalid base for `typing.NewType`
--> src/mdtest_snippet.py:11:22
|
9 | a: int
10 |
11 | Bar = NewType("Bar", Foo) # error: [invalid-newtype]
| ^^^ type `Foo`
|
info: The base of a `NewType` is not allowed to be a `TypedDict`.
info: rule `invalid-newtype` is enabled by default
```

37
crates/ty_python_semantic/resources/mdtest/snapshots/new_types.md_-_NewType_-_Trying_to_subclass_a…_(fd3c73e2a9f04).snap Normal file
View File

@@ -0,0 +1,37 @@
---
source: crates/ty_test/src/lib.rs
expression: snapshot
---
---
mdtest name: new_types.md - NewType - Trying to subclass a `NewType` produces an error matching CPython
mdtest path: crates/ty_python_semantic/resources/mdtest/annotations/new_types.md
---
# Python source files
## mdtest_snippet.py
```
1 | from typing import NewType
2 |
3 | X = NewType("X", int)
4 |
5 | class Foo(X): ... # error: [invalid-base]
```
# Diagnostics
```
error[invalid-base]: Cannot subclass an instance of NewType
--> src/mdtest_snippet.py:5:11
|
3 | X = NewType("X", int)
4 |
5 | class Foo(X): ... # error: [invalid-base]
| ^
|
info: Perhaps you were looking for: `Foo = NewType('Foo', X)`
info: Definition of class `Foo` will raise `TypeError` at runtime
info: rule `invalid-base` is enabled by default
```

152
crates/ty_python_semantic/resources/mdtest/snapshots/super.md_-_Super_-_Basic_Usage_-_Explicit_Super_Objec…_(b753048091f275c0).snap
View File

@@ -46,7 +46,7 @@ mdtest path: crates/ty_python_semantic/resources/mdtest/class/super.md
32 | reveal_type(super(C, C()).aa) # revealed: int
33 | reveal_type(super(C, C()).bb) # revealed: int
34 | import types
35 | from typing_extensions import Callable, TypeIs, Literal, TypedDict
35 | from typing_extensions import Callable, TypeIs, Literal, NewType, TypedDict
36 |
37 | def f(): ...
38 |
@@ -61,59 +61,63 @@ mdtest path: crates/ty_python_semantic/resources/mdtest/class/super.md
47 | x: int
48 | y: bytes
49 |
50 | # revealed: <super: <class 'object'>, FunctionType>
51 | reveal_type(super(object, f))
52 | # revealed: <super: <class 'object'>, WrapperDescriptorType>
53 | reveal_type(super(object, types.FunctionType.__get__))
54 | # revealed: <super: <class 'object'>, GenericAlias>
55 | reveal_type(super(object, Foo[int]))
56 | # revealed: <super: <class 'object'>, _SpecialForm>
57 | reveal_type(super(object, Literal))
58 | # revealed: <super: <class 'object'>, TypeAliasType>
59 | reveal_type(super(object, Alias))
60 | # revealed: <super: <class 'object'>, MethodType>
61 | reveal_type(super(object, Foo().method))
62 | # revealed: <super: <class 'object'>, property>
63 | reveal_type(super(object, Foo.some_property))
64 |
65 | def g(x: object) -> TypeIs[list[object]]:
66 | return isinstance(x, list)
67 |
68 | def _(x: object, y: SomeTypedDict, z: Callable[[int, str], bool]):
69 | if hasattr(x, "bar"):
70 | # revealed: <Protocol with members 'bar'>
71 | reveal_type(x)
72 | # error: [invalid-super-argument]
73 | # revealed: Unknown
74 | reveal_type(super(object, x))
75 |
76 | # error: [invalid-super-argument]
77 | # revealed: Unknown
78 | reveal_type(super(object, z))
50 | N = NewType("N", int)
51 |
52 | # revealed: <super: <class 'object'>, FunctionType>
53 | reveal_type(super(object, f))
54 | # revealed: <super: <class 'object'>, WrapperDescriptorType>
55 | reveal_type(super(object, types.FunctionType.__get__))
56 | # revealed: <super: <class 'object'>, GenericAlias>
57 | reveal_type(super(object, Foo[int]))
58 | # revealed: <super: <class 'object'>, _SpecialForm>
59 | reveal_type(super(object, Literal))
60 | # revealed: <super: <class 'object'>, TypeAliasType>
61 | reveal_type(super(object, Alias))
62 | # revealed: <super: <class 'object'>, MethodType>
63 | reveal_type(super(object, Foo().method))
64 | # revealed: <super: <class 'object'>, property>
65 | reveal_type(super(object, Foo.some_property))
66 | # revealed: <super: <class 'object'>, int>
67 | reveal_type(super(object, N(42)))
68 |
69 | def g(x: object) -> TypeIs[list[object]]:
70 | return isinstance(x, list)
71 |
72 | def _(x: object, y: SomeTypedDict, z: Callable[[int, str], bool]):
73 | if hasattr(x, "bar"):
74 | # revealed: <Protocol with members 'bar'>
75 | reveal_type(x)
76 | # error: [invalid-super-argument]
77 | # revealed: Unknown
78 | reveal_type(super(object, x))
79 |
80 | is_list = g(x)
81 | # revealed: TypeIs[list[object] @ x]
82 | reveal_type(is_list)
83 | # revealed: <super: <class 'object'>, bool>
84 | reveal_type(super(object, is_list))
85 |
86 | # revealed: <super: <class 'object'>, dict[Literal["x", "y"], int | bytes]>
87 | reveal_type(super(object, y))
88 |
89 | # The first argument to `super()` must be an actual class object;
90 | # instances of `GenericAlias` are not accepted at runtime:
91 | #
92 | # error: [invalid-super-argument]
93 | # revealed: Unknown
94 | reveal_type(super(list[int], []))
95 | class Super:
96 | def method(self) -> int:
97 | return 42
98 |
99 | class Sub(Super):
100 | def method(self: Sub) -> int:
101 | # revealed: <super: <class 'Sub'>, Sub>
102 | return reveal_type(super(self.__class__, self)).method()
80 | # error: [invalid-super-argument]
81 | # revealed: Unknown
82 | reveal_type(super(object, z))
83 |
84 | is_list = g(x)
85 | # revealed: TypeIs[list[object] @ x]
86 | reveal_type(is_list)
87 | # revealed: <super: <class 'object'>, bool>
88 | reveal_type(super(object, is_list))
89 |
90 | # revealed: <super: <class 'object'>, dict[Literal["x", "y"], int | bytes]>
91 | reveal_type(super(object, y))
92 |
93 | # The first argument to `super()` must be an actual class object;
94 | # instances of `GenericAlias` are not accepted at runtime:
95 | #
96 | # error: [invalid-super-argument]
97 | # revealed: Unknown
98 | reveal_type(super(list[int], []))
99 | class Super:
100 | def method(self) -> int:
101 | return 42
102 |
103 | class Sub(Super):
104 | def method(self: Sub) -> int:
105 | # revealed: <super: <class 'Sub'>, Sub>
106 | return reveal_type(super(self.__class__, self)).method()
```
# Diagnostics
@@ -206,14 +210,14 @@ info: rule `unresolved-attribute` is enabled by default
```
error[invalid-super-argument]: `<Protocol with members 'bar'>` is an abstract/structural type in `super(<class 'object'>, <Protocol with members 'bar'>)` call
--> src/mdtest_snippet.py:74:21
--> src/mdtest_snippet.py:78:21
|
72 | # error: [invalid-super-argument]
73 | # revealed: Unknown
74 | reveal_type(super(object, x))
76 | # error: [invalid-super-argument]
77 | # revealed: Unknown
78 | reveal_type(super(object, x))
| ^^^^^^^^^^^^^^^^
75 |
76 | # error: [invalid-super-argument]
79 |
80 | # error: [invalid-super-argument]
|
info: rule `invalid-super-argument` is enabled by default
@@ -221,14 +225,14 @@ info: rule `invalid-super-argument` is enabled by default
```
error[invalid-super-argument]: `(int, str, /) -> bool` is an abstract/structural type in `super(<class 'object'>, (int, str, /) -> bool)` call
--> src/mdtest_snippet.py:78:17
--> src/mdtest_snippet.py:82:17
|
76 | # error: [invalid-super-argument]
77 | # revealed: Unknown
78 | reveal_type(super(object, z))
80 | # error: [invalid-super-argument]
81 | # revealed: Unknown
82 | reveal_type(super(object, z))
| ^^^^^^^^^^^^^^^^
79 |
80 | is_list = g(x)
83 |
84 | is_list = g(x)
|
info: rule `invalid-super-argument` is enabled by default
@@ -236,15 +240,15 @@ info: rule `invalid-super-argument` is enabled by default
```
error[invalid-super-argument]: `types.GenericAlias` instance `list[int]` is not a valid class
--> src/mdtest_snippet.py:94:13
|
92 | # error: [invalid-super-argument]
93 | # revealed: Unknown
94 | reveal_type(super(list[int], []))
| ^^^^^^^^^^^^^^^^^^^^
95 | class Super:
96 | def method(self) -> int:
|
--> src/mdtest_snippet.py:98:13
|
96 | # error: [invalid-super-argument]
97 | # revealed: Unknown
98 | reveal_type(super(list[int], []))
| ^^^^^^^^^^^^^^^^^^^^
99 | class Super:
100 | def method(self) -> int:
|
info: rule `invalid-super-argument` is enabled by default
```

151
crates/ty_python_semantic/src/types.rs
View File

@@ -66,6 +66,7 @@ use crate::types::generics::{
use crate::types::infer::infer_unpack_types;
use crate::types::mro::{Mro, MroError, MroIterator};
pub(crate) use crate::types::narrow::infer_narrowing_constraint;
use crate::types::newtype::NewType;
use crate::types::signatures::{ParameterForm, walk_signature};
use crate::types::tuple::{TupleSpec, TupleSpecBuilder};
pub(crate) use crate::types::typed_dict::{TypedDictParams, TypedDictType, walk_typed_dict_type};
@@ -98,6 +99,7 @@ mod instance;
mod member;
mod mro;
mod narrow;
mod newtype;
mod protocol_class;
mod signatures;
mod special_form;
@@ -783,6 +785,13 @@ pub enum Type<'db> {
TypedDict(TypedDictType<'db>),
/// An aliased type (lazily not-yet-unpacked to its value type).
TypeAlias(TypeAliasType<'db>),
/// The set of Python objects that belong to a `typing.NewType` subtype. Note that
/// `typing.NewType` itself is a `Type::ClassLiteral` with `KnownClass::NewType`, and the
/// identity callables it returns (which behave like subtypes in type expressions) are of
/// `Type::KnownInstance` with `KnownInstanceType::NewType`. This `Type` refers to the objects
/// wrapped/returned by a specific one of those identity callables, or by another that inherits
/// from it.
NewTypeInstance(NewType<'db>),
}
#[salsa::tracked]
@@ -1420,6 +1429,13 @@ impl<'db> Type<'db> {
self
}
Type::TypeAlias(alias) => alias.value_type(db).normalized_impl(db, visitor),
Type::NewTypeInstance(newtype) => {
visitor.visit(self, || {
Type::NewTypeInstance(newtype.map_base_class_type(db, |class_type| {
class_type.normalized_impl(db, visitor)
}))
})
}
Type::LiteralString
| Type::AlwaysFalsy
| Type::AlwaysTruthy
@@ -1482,7 +1498,8 @@ impl<'db> Type<'db> {
| Type::BoundSuper(_)
| Type::TypeIs(_)
| Type::TypedDict(_)
| Type::TypeAlias(_) => false,
| Type::TypeAlias(_)
| Type::NewTypeInstance(_) => false,
}
}
@@ -1520,6 +1537,10 @@ impl<'db> Type<'db> {
Type::GenericAlias(alias) => Some(ClassType::Generic(alias).into_callable(db)),
Type::NewTypeInstance(newtype) => {
Type::instance(db, newtype.base_class_type(db)).try_upcast_to_callable(db)
}
// TODO: This is unsound so in future we can consider an opt-in option to disable it.
Type::SubclassOf(subclass_of_ty) => match subclass_of_ty.subclass_of() {
SubclassOfInner::Class(class) => Some(class.into_callable(db)),
@@ -1549,6 +1570,15 @@ impl<'db> Type<'db> {
false,
))),
Type::KnownInstance(KnownInstanceType::NewType(newtype)) => Some(CallableType::single(
db,
Signature::new(
Parameters::new([Parameter::positional_only(None)
.with_annotated_type(newtype.base(db).instance_type(db))]),
Some(Type::NewTypeInstance(newtype)),
),
)),
Type::Never
| Type::DataclassTransformer(_)
| Type::AlwaysTruthy
@@ -2429,6 +2459,22 @@ impl<'db> Type<'db> {
})
}
(Type::NewTypeInstance(self_newtype), Type::NewTypeInstance(target_newtype)) => {
self_newtype.has_relation_to_impl(db, target_newtype)
}
(
Type::NewTypeInstance(self_newtype),
Type::NominalInstance(target_nominal_instance),
) => self_newtype.base_class_type(db).has_relation_to_impl(
db,
target_nominal_instance.class(db),
inferable,
relation,
relation_visitor,
disjointness_visitor,
),
(Type::PropertyInstance(_), _) => {
KnownClass::Property.to_instance(db).has_relation_to_impl(
db,
@@ -2448,14 +2494,15 @@ impl<'db> Type<'db> {
disjointness_visitor,
),
// Other than the special cases enumerated above, `Instance` types and typevars are
// never subtypes of any other variants
// Other than the special cases enumerated above, nominal-instance types,
// newtype-instance types, and typevars are never subtypes of any other variants
(Type::TypeVar(bound_typevar), _) => {
// All inferable cases should have been handled above
assert!(!bound_typevar.is_inferable(db, inferable));
ConstraintSet::from(false)
}
(Type::NominalInstance(_), _) => ConstraintSet::from(false),
(Type::NewTypeInstance(_), _) => ConstraintSet::from(false),
}
}
@@ -2529,6 +2576,10 @@ impl<'db> Type<'db> {
})
}
(Type::NewTypeInstance(self_newtype), Type::NewTypeInstance(other_newtype)) => {
ConstraintSet::from(self_newtype.is_equivalent_to_impl(db, other_newtype))
}
(Type::NominalInstance(first), Type::NominalInstance(second)) => {
first.is_equivalent_to_impl(db, second, inferable, visitor)
}
@@ -3288,6 +3339,19 @@ impl<'db> Type<'db> {
)
}),
(Type::NewTypeInstance(left), Type::NewTypeInstance(right)) => {
left.is_disjoint_from_impl(db, right)
}
(Type::NewTypeInstance(newtype), other) | (other, Type::NewTypeInstance(newtype)) => {
Type::instance(db, newtype.base_class_type(db)).is_disjoint_from_impl(
db,
other,
inferable,
disjointness_visitor,
relation_visitor,
)
}
(Type::PropertyInstance(_), other) | (other, Type::PropertyInstance(_)) => {
KnownClass::Property.to_instance(db).is_disjoint_from_impl(
db,
@@ -3432,6 +3496,9 @@ impl<'db> Type<'db> {
Type::TypeIs(type_is) => type_is.is_bound(db),
Type::TypedDict(_) => false,
Type::TypeAlias(alias) => alias.value_type(db).is_singleton(db),
Type::NewTypeInstance(newtype) => {
Type::instance(db, newtype.base_class_type(db)).is_singleton(db)
}
}
}
@@ -3482,6 +3549,9 @@ impl<'db> Type<'db> {
}
Type::NominalInstance(instance) => instance.is_single_valued(db),
Type::NewTypeInstance(newtype) => {
Type::instance(db, newtype.base_class_type(db)).is_single_valued(db)
}
Type::BoundSuper(_) => {
// At runtime two super instances never compare equal, even if their arguments are identical.
@@ -3645,7 +3715,8 @@ impl<'db> Type<'db> {
| Type::ProtocolInstance(_)
| Type::PropertyInstance(_)
| Type::TypeIs(_)
| Type::TypedDict(_) => None,
| Type::TypedDict(_)
| Type::NewTypeInstance(_) => None,
}
}
@@ -3732,6 +3803,7 @@ impl<'db> Type<'db> {
Type::Dynamic(_) | Type::Never => Place::bound(self).into(),
Type::NominalInstance(instance) => instance.class(db).instance_member(db, name),
Type::NewTypeInstance(newtype) => newtype.base_class_type(db).instance_member(db, name),
Type::ProtocolInstance(protocol) => protocol.instance_member(db, name),
@@ -4404,6 +4476,7 @@ impl<'db> Type<'db> {
Type::NominalInstance(..)
| Type::ProtocolInstance(..)
| Type::NewTypeInstance(..)
| Type::BooleanLiteral(..)
| Type::IntLiteral(..)
| Type::StringLiteral(..)
@@ -4842,6 +4915,8 @@ impl<'db> Type<'db> {
.value_type(db)
.try_bool_impl(db, allow_short_circuit, visitor)
})?,
Type::NewTypeInstance(newtype) => Type::instance(db, newtype.base_class_type(db))
.try_bool_impl(db, allow_short_circuit, visitor)?,
};
Ok(truthiness)
@@ -5528,7 +5603,7 @@ impl<'db> Type<'db> {
SubclassOfInner::Class(class) => Type::from(class).bindings(db),
},
Type::NominalInstance(_) | Type::ProtocolInstance(_) => {
Type::NominalInstance(_) | Type::ProtocolInstance(_) | Type::NewTypeInstance(_) => {
// Note that for objects that have a (possibly not callable!) `__call__` attribute,
// we will get the signature of the `__call__` attribute, but will pass in the type
// of the original object as the "callable type". That ensures that we get errors
@@ -5581,6 +5656,16 @@ impl<'db> Type<'db> {
Type::EnumLiteral(enum_literal) => enum_literal.enum_class_instance(db).bindings(db),
Type::KnownInstance(KnownInstanceType::NewType(newtype)) => Binding::single(
self,
Signature::new(
Parameters::new([Parameter::positional_only(None)
.with_annotated_type(newtype.base(db).instance_type(db))]),
Some(Type::NewTypeInstance(newtype)),
),
)
.into(),
Type::KnownInstance(known_instance) => {
known_instance.instance_fallback(db).bindings(db)
}
@@ -5716,6 +5801,7 @@ impl<'db> Type<'db> {
match ty {
Type::NominalInstance(nominal) => nominal.tuple_spec(db),
Type::NewTypeInstance(newtype) => non_async_special_case(db, Type::instance(db, newtype.base_class_type(db))),
Type::GenericAlias(alias) if alias.origin(db).is_tuple(db) => {
Some(Cow::Owned(TupleSpec::homogeneous(todo_type!(
"*tuple[] annotations"
@@ -6346,6 +6432,9 @@ impl<'db> Type<'db> {
Type::ClassLiteral(class) => Some(Type::instance(db, class.default_specialization(db))),
Type::GenericAlias(alias) => Some(Type::instance(db, ClassType::from(alias))),
Type::SubclassOf(subclass_of_ty) => Some(subclass_of_ty.to_instance(db)),
Type::KnownInstance(KnownInstanceType::NewType(newtype)) => {
Some(Type::NewTypeInstance(newtype))
}
Type::Union(union) => union.to_instance(db),
// If there is no bound or constraints on a typevar `T`, `T: object` implicitly, which
// has no instance type. Otherwise, synthesize a typevar with bound or constraints
@@ -6376,7 +6465,8 @@ impl<'db> Type<'db> {
| Type::AlwaysTruthy
| Type::AlwaysFalsy
| Type::TypeIs(_)
| Type::TypedDict(_) => None,
| Type::TypedDict(_)
| Type::NewTypeInstance(_) => None,
}
}
@@ -6455,6 +6545,7 @@ impl<'db> Type<'db> {
Type::KnownInstance(known_instance) => match known_instance {
KnownInstanceType::TypeAliasType(alias) => Ok(Type::TypeAlias(*alias)),
KnownInstanceType::NewType(newtype) => Ok(Type::NewTypeInstance(*newtype)),
KnownInstanceType::TypeVar(typevar) => {
let index = semantic_index(db, scope_id.file(db));
Ok(bind_typevar(
@@ -6669,9 +6760,6 @@ impl<'db> Type<'db> {
Some(KnownClass::TypeVarTuple) => Ok(todo_type!(
"Support for `typing.TypeVarTuple` instances in type expressions"
)),
Some(KnownClass::NewType) => Ok(todo_type!(
"Support for `typing.NewType` instances in type expressions"
)),
Some(KnownClass::GenericAlias) => Ok(todo_type!(
"Support for `typing.GenericAlias` instances in type expressions"
)),
@@ -6690,6 +6778,13 @@ impl<'db> Type<'db> {
.value_type(db)
.in_type_expression(db, scope_id, typevar_binding_context)
}
Type::NewTypeInstance(_) => Err(InvalidTypeExpressionError {
invalid_expressions: smallvec::smallvec_inline![
InvalidTypeExpression::InvalidType(*self, scope_id)
],
fallback_type: Type::unknown(),
}),
}
}
@@ -6764,6 +6859,7 @@ impl<'db> Type<'db> {
// understand a more specific meta type in order to correctly handle `__getitem__`.
Type::TypedDict(typed_dict) => SubclassOfType::from(db, typed_dict.defining_class()),
Type::TypeAlias(alias) => alias.value_type(db).to_meta_type(db),
Type::NewTypeInstance(newtype) => Type::from(newtype.base_class_type(db)),
}
}
@@ -6873,8 +6969,8 @@ impl<'db> Type<'db> {
| TypeMapping::ReplaceParameterDefaults
| TypeMapping::BindLegacyTypevars(_) => self,
TypeMapping::Materialize(materialization_kind) => {
Type::TypeVar(bound_typevar.materialize_impl(db, *materialization_kind, visitor))
}
Type::TypeVar(bound_typevar.materialize_impl(db, *materialization_kind, visitor))
}
}
Type::KnownInstance(KnownInstanceType::TypeVar(typevar)) => match type_mapping {
@@ -6909,6 +7005,12 @@ impl<'db> Type<'db> {
instance.apply_type_mapping_impl(db, type_mapping, tcx, visitor)
},
Type::NewTypeInstance(newtype) => visitor.visit(self, || {
Type::NewTypeInstance(newtype.map_base_class_type(db, |class_type| {
class_type.apply_type_mapping_impl(db, type_mapping, tcx, visitor)
}))
}),
Type::ProtocolInstance(instance) => {
// TODO: Add tests for materialization once subtyping/assignability is implemented for
// protocols. It _might_ require changing the logic here because:
@@ -7150,6 +7252,12 @@ impl<'db> Type<'db> {
instance.find_legacy_typevars_impl(db, binding_context, typevars, visitor);
}
Type::NewTypeInstance(_) => {
// A newtype can never be constructed from an unspecialized generic class, so it is
// impossible that we could ever find any legacy typevars in a newtype instance or
// its underlying class.
}
Type::SubclassOf(subclass_of) => {
subclass_of.find_legacy_typevars_impl(db, binding_context, typevars, visitor);
}
@@ -7305,6 +7413,7 @@ impl<'db> Type<'db> {
},
Self::TypeAlias(alias) => alias.value_type(db).definition(db),
Self::NewTypeInstance(newtype) => Some(TypeDefinition::NewType(newtype.definition(db))),
Self::StringLiteral(_)
| Self::BooleanLiteral(_)
@@ -7528,7 +7637,8 @@ impl<'db> VarianceInferable<'db> for Type<'db> {
| Type::BoundSuper(_)
| Type::TypeVar(_)
| Type::TypedDict(_)
| Type::TypeAlias(_) => TypeVarVariance::Bivariant,
| Type::TypeAlias(_)
| Type::NewTypeInstance(_) => TypeVarVariance::Bivariant,
};
tracing::trace!(
@@ -7726,6 +7836,10 @@ pub enum KnownInstanceType<'db> {
/// A single instance of `typing.Annotated`
Annotated(InternedType<'db>),
/// An identity callable created with `typing.NewType(name, base)`, which behaves like a
/// subtype of `base` in type expressions. See the `struct NewType` payload for an example.
NewType(NewType<'db>),
}
fn walk_known_instance_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>(
@@ -7760,6 +7874,11 @@ fn walk_known_instance_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>(
KnownInstanceType::Literal(ty) | KnownInstanceType::Annotated(ty) => {
visitor.visit_type(db, ty.inner(db));
}
KnownInstanceType::NewType(newtype) => {
if let ClassType::Generic(generic_alias) = newtype.base_class_type(db) {
visitor.visit_generic_alias_type(db, generic_alias);
}
}
}
}
@@ -7799,6 +7918,10 @@ impl<'db> KnownInstanceType<'db> {
Self::UnionType(list) => Self::UnionType(list.normalized_impl(db, visitor)),
Self::Literal(ty) => Self::Literal(ty.normalized_impl(db, visitor)),
Self::Annotated(ty) => Self::Annotated(ty.normalized_impl(db, visitor)),
Self::NewType(newtype) => Self::NewType(
newtype
.map_base_class_type(db, |class_type| class_type.normalized_impl(db, visitor)),
),
}
}
@@ -7819,6 +7942,7 @@ impl<'db> KnownInstanceType<'db> {
Self::UnionType(_) => KnownClass::UnionType,
Self::Literal(_) => KnownClass::GenericAlias,
Self::Annotated(_) => KnownClass::GenericAlias,
Self::NewType(_) => KnownClass::NewType,
}
}
@@ -7903,6 +8027,9 @@ impl<'db> KnownInstanceType<'db> {
KnownInstanceType::Annotated(_) => {
f.write_str("<typing.Annotated special form>")
}
KnownInstanceType::NewType(declaration) => {
write!(f, "<NewType pseudo-class '{}'>", declaration.name(self.db))
}
}
}
}

3
crates/ty_python_semantic/src/types/bound_super.rs
View File

@@ -404,6 +404,9 @@ impl<'db> BoundSuperType<'db> {
.to_specialized_instance(db, [key_builder.build(), value_builder.build()]),
);
}
Type::NewTypeInstance(newtype) => {
return delegate_to(Type::instance(db, newtype.base_class_type(db)));
}
Type::Callable(callable) if callable.is_function_like(db) => {
return delegate_to(KnownClass::FunctionType.to_instance(db));
}

8
crates/ty_python_semantic/src/types/class.rs
View File

@@ -358,6 +358,14 @@ pub enum ClassType<'db> {
#[salsa::tracked]
impl<'db> ClassType<'db> {
/// Return a `ClassType` representing the class `builtins.object`
pub(super) fn object(db: &'db dyn Db) -> Self {
KnownClass::Object
.to_class_literal(db)
.to_class_type(db)
.unwrap()
}
pub(super) const fn is_generic(self) -> bool {
matches!(self, Self::Generic(_))
}

12
crates/ty_python_semantic/src/types/class_base.rs
View File

@@ -137,6 +137,12 @@ impl<'db> ClassBase<'db> {
Type::TypeAlias(alias) => Self::try_from_type(db, alias.value_type(db), subclass),
Type::NewTypeInstance(newtype) => ClassBase::try_from_type(
db,
Type::instance(db, newtype.base_class_type(db)),
subclass,
),
Type::PropertyInstance(_)
| Type::BooleanLiteral(_)
| Type::FunctionLiteral(_)
@@ -169,7 +175,11 @@ impl<'db> ClassBase<'db> {
| KnownInstanceType::Field(_)
| KnownInstanceType::ConstraintSet(_)
| KnownInstanceType::UnionType(_)
| KnownInstanceType::Literal(_) => None,
| KnownInstanceType::Literal(_)
// A class inheriting from a newtype would make intuitive sense, but newtype
// wrappers are just identity callables at runtime, so this sort of inheritance
// doesn't work and isn't allowed.
| KnownInstanceType::NewType(_) => None,
KnownInstanceType::Annotated(ty) => Self::try_from_type(db, ty.inner(db), subclass),
},

7
crates/ty_python_semantic/src/types/definition.rs
View File

@@ -12,6 +12,7 @@ pub enum TypeDefinition<'db> {
Function(Definition<'db>),
TypeVar(Definition<'db>),
TypeAlias(Definition<'db>),
NewType(Definition<'db>),
}
impl TypeDefinition<'_> {
@@ -21,7 +22,8 @@ impl TypeDefinition<'_> {
Self::Class(definition)
| Self::Function(definition)
| Self::TypeVar(definition)
| Self::TypeAlias(definition) => {
| Self::TypeAlias(definition)
| Self::NewType(definition) => {
let module = parsed_module(db, definition.file(db)).load(db);
Some(definition.focus_range(db, &module))
}
@@ -38,7 +40,8 @@ impl TypeDefinition<'_> {
Self::Class(definition)
| Self::Function(definition)
| Self::TypeVar(definition)
| Self::TypeAlias(definition) => {
| Self::TypeAlias(definition)
| Self::NewType(definition) => {
let module = parsed_module(db, definition.file(db)).load(db);
Some(definition.full_range(db, &module))
}

44
crates/ty_python_semantic/src/types/diagnostic.rs
View File

@@ -12,6 +12,7 @@ use crate::semantic_index::definition::{Definition, DefinitionKind};
use crate::semantic_index::place::{PlaceTable, ScopedPlaceId};
use crate::semantic_index::{global_scope, place_table};
use crate::suppression::FileSuppressionId;
use crate::types::KnownInstanceType;
use crate::types::call::CallError;
use crate::types::class::{DisjointBase, DisjointBaseKind, Field};
use crate::types::function::KnownFunction;
@@ -65,6 +66,7 @@ pub(crate) fn register_lints(registry: &mut LintRegistryBuilder) {
registry.register_lint(&INVALID_LEGACY_TYPE_VARIABLE);
registry.register_lint(&INVALID_PARAMSPEC);
registry.register_lint(&INVALID_TYPE_ALIAS_TYPE);
registry.register_lint(&INVALID_NEWTYPE);
registry.register_lint(&INVALID_METACLASS);
registry.register_lint(&INVALID_OVERLOAD);
registry.register_lint(&USELESS_OVERLOAD_BODY);
@@ -926,6 +928,30 @@ declare_lint! {
}
}
declare_lint! {
/// ## What it does
/// Checks for the creation of invalid `NewType`s
///
/// ## Why is this bad?
/// There are several requirements that you must follow when creating a `NewType`.
///
/// ## Examples
/// ```python
/// from typing import NewType
///
/// def get_name() -> str: ...
///
/// Foo = NewType("Foo", int) # okay
/// Bar = NewType(get_name(), int) # error: The first argument to `NewType` must be a string literal
/// Baz = NewType("Baz", int | str) # error: invalid base for `typing.NewType`
/// ```
pub(crate) static INVALID_NEWTYPE = {
summary: "detects invalid NewType definitions",
status: LintStatus::preview("1.0.0"),
default_level: Level::Error,
}
}
declare_lint! {
/// ## What it does
/// Checks for arguments to `metaclass=` that are invalid.
@@ -2898,6 +2924,24 @@ pub(crate) fn report_invalid_or_unsupported_base(
return;
}
if let Type::KnownInstance(KnownInstanceType::NewType(newtype)) = base_type {
let Some(builder) = context.report_lint(&INVALID_BASE, base_node) else {
return;
};
let mut diagnostic = builder.into_diagnostic("Cannot subclass an instance of NewType");
diagnostic.info(format_args!(
"Perhaps you were looking for: `{} = NewType('{}', {})`",
class.name(context.db()),
class.name(context.db()),
newtype.name(context.db()),
));
diagnostic.info(format_args!(
"Definition of class `{}` will raise `TypeError` at runtime",
class.name(context.db())
));
return;
}
let tuple_of_types = Type::homogeneous_tuple(db, instance_of_type);
let explain_mro_entries = |diagnostic: &mut LintDiagnosticGuard| {

1
crates/ty_python_semantic/src/types/display.rs
View File

@@ -618,6 +618,7 @@ impl Display for DisplayRepresentation<'_> {
.fmt(f),
}
}
Type::NewTypeInstance(newtype) => f.write_str(newtype.name(self.db)),
}
}
}

5
crates/ty_python_semantic/src/types/function.rs
View File

@@ -1101,6 +1101,11 @@ fn is_instance_truthiness<'db>(
Type::NominalInstance(..) => always_true_if(is_instance(&ty)),
Type::NewTypeInstance(newtype) => always_true_if(is_instance(&Type::instance(
db,
newtype.base_class_type(db),
))),
Type::BooleanLiteral(..)
| Type::BytesLiteral(..)
| Type::IntLiteral(..)

4
crates/ty_python_semantic/src/types/ide_support.rs
View File

@@ -128,6 +128,10 @@ impl<'db> AllMembers<'db> {
}
}
Type::NewTypeInstance(newtype) => {
self.extend_with_type(db, Type::instance(db, newtype.base_class_type(db)));
}
Type::ClassLiteral(class_literal) if class_literal.is_typed_dict(db) => {
self.extend_with_type(db, KnownClass::TypedDictFallback.to_class_literal(db));
}

176
crates/ty_python_semantic/src/types/infer/builder.rs
View File

@@ -59,8 +59,8 @@ use crate::types::diagnostic::{
DIVISION_BY_ZERO, DUPLICATE_KW_ONLY, INCONSISTENT_MRO, INVALID_ARGUMENT_TYPE,
INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, INVALID_BASE, INVALID_DECLARATION,
INVALID_GENERIC_CLASS, INVALID_KEY, INVALID_LEGACY_TYPE_VARIABLE, INVALID_METACLASS,
INVALID_NAMED_TUPLE, INVALID_OVERLOAD, INVALID_PARAMETER_DEFAULT, INVALID_PARAMSPEC,
INVALID_PROTOCOL, INVALID_TYPE_FORM, INVALID_TYPE_GUARD_CALL,
INVALID_NAMED_TUPLE, INVALID_NEWTYPE, INVALID_OVERLOAD, INVALID_PARAMETER_DEFAULT,
INVALID_PARAMSPEC, INVALID_PROTOCOL, INVALID_TYPE_FORM, INVALID_TYPE_GUARD_CALL,
INVALID_TYPE_VARIABLE_CONSTRAINTS, IncompatibleBases, NON_SUBSCRIPTABLE,
POSSIBLY_MISSING_IMPLICIT_CALL, POSSIBLY_MISSING_IMPORT, SUBCLASS_OF_FINAL_CLASS,
UNDEFINED_REVEAL, UNRESOLVED_ATTRIBUTE, UNRESOLVED_GLOBAL, UNRESOLVED_IMPORT,
@@ -90,6 +90,7 @@ use crate::types::generics::{
use crate::types::infer::nearest_enclosing_function;
use crate::types::instance::SliceLiteral;
use crate::types::mro::MroErrorKind;
use crate::types::newtype::NewType;
use crate::types::signatures::Signature;
use crate::types::subclass_of::SubclassOfInner;
use crate::types::tuple::{Tuple, TupleLength, TupleSpec, TupleType};
@@ -3884,7 +3885,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
| Type::AlwaysTruthy
| Type::AlwaysFalsy
| Type::TypeIs(_)
| Type::TypedDict(_) => {
| Type::TypedDict(_)
| Type::NewTypeInstance(_) => {
// TODO: We could use the annotated parameter type of `__setattr__` as type context here.
// However, we would still have to perform the first inference without type context.
let value_ty = infer_value_ty(self, TypeContext::default());
@@ -4454,6 +4456,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
Some(KnownClass::ParamSpec) => {
self.infer_paramspec(target, call_expr, definition)
}
Some(KnownClass::NewType) => {
self.infer_newtype_expression(target, call_expr, definition)
}
Some(_) | None => {
self.infer_call_expression_impl(call_expr, callable_type, tcx)
}
@@ -4892,14 +4897,114 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
)))
}
fn infer_newtype_expression(
&mut self,
target: &ast::Expr,
call_expr: &ast::ExprCall,
definition: Definition<'db>,
) -> Type<'db> {
fn error<'db>(
context: &InferContext<'db, '_>,
message: impl std::fmt::Display,
node: impl Ranged,
) -> Type<'db> {
if let Some(builder) = context.report_lint(&INVALID_NEWTYPE, node) {
builder.into_diagnostic(message);
}
Type::unknown()
}
let db = self.db();
let arguments = &call_expr.arguments;
if !arguments.keywords.is_empty() {
return error(
&self.context,
"Keyword arguments are not supported in `NewType` creation",
call_expr,
);
}
if let Some(starred) = arguments.args.iter().find(|arg| arg.is_starred_expr()) {
return error(
&self.context,
"Starred arguments are not supported in `NewType` creation",
starred,
);
}
if arguments.args.len() != 2 {
return error(
&self.context,
format!(
"Wrong number of arguments in `NewType` creation, expected 2, found {}",
arguments.args.len()
),
call_expr,
);
}
let name_param_ty = self.infer_expression(&arguments.args[0], TypeContext::default());
let Some(name) = name_param_ty.as_string_literal().map(|name| name.value(db)) else {
return error(
&self.context,
"The first argument to `NewType` must be a string literal",
call_expr,
);
};
let ast::Expr::Name(ast::ExprName {
id: target_name, ..
}) = target
else {
return error(
&self.context,
"A `NewType` definition must be a simple variable assignment",
target,
);
};
if name != target_name {
return error(
&self.context,
format_args!(
"The name of a `NewType` (`{name}`) must match \
the name of the variable it is assigned to (`{target_name}`)"
),
target,
);
}
// Inference of `tp` must be deferred, to avoid cycles.
self.deferred.insert(definition, self.multi_inference_state);
Type::KnownInstance(KnownInstanceType::NewType(NewType::new(
db,
ast::name::Name::from(name),
definition,
None,
)))
}
fn infer_assignment_deferred(&mut self, value: &ast::Expr) {
// Infer deferred bounds/constraints/defaults of a legacy TypeVar / ParamSpec.
// Infer deferred bounds/constraints/defaults of a legacy TypeVar / ParamSpec / NewType.
let ast::Expr::Call(ast::ExprCall {
func, arguments, ..
}) = value
else {
return;
};
let func_ty = self
.try_expression_type(func)
.unwrap_or_else(|| self.infer_expression(func, TypeContext::default()));
let known_class = func_ty
.as_class_literal()
.and_then(|cls| cls.known(self.db()));
if let Some(KnownClass::NewType) = known_class {
self.infer_newtype_assignment_deferred(arguments);
return;
}
for arg in arguments.args.iter().skip(1) {
self.infer_type_expression(arg);
}
@@ -4907,12 +5012,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
self.infer_type_expression(&bound.value);
}
if let Some(default) = arguments.find_keyword("default") {
let func_ty = self
.try_expression_type(func)
.unwrap_or_else(|| self.infer_expression(func, TypeContext::default()));
if func_ty.as_class_literal().is_some_and(|class_literal| {
class_literal.is_known(self.db(), KnownClass::ParamSpec)
}) {
if let Some(KnownClass::ParamSpec) = known_class {
self.infer_paramspec_default(&default.value);
} else {
self.infer_type_expression(&default.value);
@@ -4920,6 +5020,34 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
}
}
// Infer the deferred base type of a NewType.
fn infer_newtype_assignment_deferred(&mut self, arguments: &ast::Arguments) {
match self.infer_type_expression(&arguments.args[1]) {
Type::NominalInstance(_) | Type::NewTypeInstance(_) => {}
// `Unknown` is likely to be the result of an unresolved import or a typo, which will
// already get a diagnostic, so don't pile on an extra diagnostic here.
Type::Dynamic(DynamicType::Unknown) => {}
other_type => {
if let Some(builder) = self
.context
.report_lint(&INVALID_NEWTYPE, &arguments.args[1])
{
let mut diag = builder.into_diagnostic("invalid base for `typing.NewType`");
diag.set_primary_message(format!("type `{}`", other_type.display(self.db())));
if matches!(other_type, Type::ProtocolInstance(_)) {
diag.info("The base of a `NewType` is not allowed to be a protocol class.");
} else if matches!(other_type, Type::TypedDict(_)) {
diag.info("The base of a `NewType` is not allowed to be a `TypedDict`.");
} else {
diag.info(
"The base of a `NewType` must be a class type or another `NewType`.",
);
}
}
}
}
}
fn infer_annotated_assignment_statement(&mut self, assignment: &ast::StmtAnnAssign) {
if assignment.target.is_name_expr() {
self.infer_definition(assignment);
@@ -7483,11 +7611,11 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
.to_class_type(self.db())
.is_none_or(|enum_class| !class.is_subclass_of(self.db(), enum_class))
{
// Inference of correctly-placed `TypeVar` and `ParamSpec` definitions is done in
// `TypeInferenceBuilder::infer_legacy_typevar` and
// `TypeInferenceBuilder::infer_paramspec`, and doesn't use the full
// call-binding machinery. If we reach here, it means that someone is trying to
// instantiate a `typing.TypeVar` and `typing.ParamSpec` in an invalid context.
// Inference of correctly-placed `TypeVar`, `ParamSpec`, and `NewType` definitions
// is done in `infer_legacy_typevar`, `infer_paramspec`, and
// `infer_newtype_expression`, and doesn't use the full call-binding machinery. If
// we reach here, it means that someone is trying to instantiate one of these in an
// invalid context.
match class.known(self.db()) {
Some(KnownClass::TypeVar | KnownClass::ExtensionsTypeVar) => {
if let Some(builder) = self
@@ -7509,6 +7637,15 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
);
}
}
Some(KnownClass::NewType) => {
if let Some(builder) =
self.context.report_lint(&INVALID_NEWTYPE, call_expression)
{
builder.into_diagnostic(
"A `NewType` definition must be a simple variable assignment",
);
}
}
_ => {}
}
@@ -8577,7 +8714,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
| Type::BoundSuper(_)
| Type::TypeVar(_)
| Type::TypeIs(_)
| Type::TypedDict(_),
| Type::TypedDict(_)
| Type::NewTypeInstance(_),
) => {
let unary_dunder_method = match op {
ast::UnaryOp::Invert => "__invert__",
@@ -9025,7 +9163,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
| Type::BoundSuper(_)
| Type::TypeVar(_)
| Type::TypeIs(_)
| Type::TypedDict(_),
| Type::TypedDict(_)
| Type::NewTypeInstance(_),
Type::FunctionLiteral(_)
| Type::BooleanLiteral(_)
| Type::Callable(..)
@@ -9054,7 +9193,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
| Type::BoundSuper(_)
| Type::TypeVar(_)
| Type::TypeIs(_)
| Type::TypedDict(_),
| Type::TypedDict(_)
| Type::NewTypeInstance(_),
op,
) => Type::try_call_bin_op(self.db(), left_ty, op, right_ty)
.map(|outcome| outcome.return_type(self.db()))

10
crates/ty_python_semantic/src/types/infer/builder/type_expression.rs
View File

@@ -828,6 +828,16 @@ impl<'db> TypeInferenceBuilder<'db, '_> {
self.infer_type_expression(slice);
todo_type!("Generic specialization of typing.Annotated")
}
KnownInstanceType::NewType(newtype) => {
self.infer_type_expression(&subscript.slice);
if let Some(builder) = self.context.report_lint(&INVALID_TYPE_FORM, subscript) {
builder.into_diagnostic(format_args!(
"`{}` is a `NewType` and cannot be specialized",
newtype.name(self.db())
));
}
Type::unknown()
}
},
Type::Dynamic(DynamicType::Todo(_)) => {
self.infer_type_expression(slice);

3
crates/ty_python_semantic/src/types/narrow.rs
View File

@@ -252,7 +252,8 @@ impl ClassInfoConstraintFunction {
| Type::TypeIs(_)
| Type::WrapperDescriptor(_)
| Type::DataclassTransformer(_)
| Type::TypedDict(_) => None,
| Type::TypedDict(_)
| Type::NewTypeInstance(_) => None,
}
}
}

266
crates/ty_python_semantic/src/types/newtype.rs Normal file
View File

@@ -0,0 +1,266 @@
use std::collections::BTreeSet;
use crate::Db;
use crate::semantic_index::definition::{Definition, DefinitionKind};
use crate::types::constraints::ConstraintSet;
use crate::types::{ClassType, Type, definition_expression_type, visitor};
use ruff_db::parsed::parsed_module;
use ruff_python_ast as ast;
/// A `typing.NewType` declaration, either from the perspective of the
/// identity-callable-that-acts-like-a-subtype-in-type-expressions returned by the call to
/// `typing.NewType(...)`, or from the perspective of instances of that subtype returned by the
/// identity callable. For example:
///
/// ```py
/// import typing
/// Foo = typing.NewType("Foo", int)
/// x = Foo(42)
/// ```
///
/// The revealed types there are:
/// - `typing.NewType`: `Type::ClassLiteral(ClassLiteral)` with `KnownClass::NewType`.
/// - `Foo`: `Type::KnownInstance(KnownInstanceType::NewType(NewType { .. }))`
/// - `x`: `Type::NewTypeInstance(NewType { .. })`
///
/// # Ordering
/// Ordering is based on the newtype's salsa-assigned id and not on its values.
/// The id may change between runs, or when the newtype was garbage collected and recreated.
#[salsa::interned(debug, heap_size=ruff_memory_usage::heap_size)]
#[derive(PartialOrd, Ord)]
pub struct NewType<'db> {
/// The name of this NewType (e.g. `"Foo"`)
#[returns(ref)]
pub name: ast::name::Name,
/// The binding where this NewType is first created.
pub definition: Definition<'db>,
// The base type of this NewType, if it's eagerly specified. This is typically `None` when a
// `NewType` is first encountered, because the base type is lazy/deferred to avoid panics in
// the recursive case. This becomes `Some` when a `NewType` is modified by methods like
// `.normalize()`. Callers should use the `base` method instead of accessing this field
// directly.
eager_base: Option<NewTypeBase<'db>>,
}
impl get_size2::GetSize for NewType<'_> {}
#[salsa::tracked]
impl<'db> NewType<'db> {
pub fn base(self, db: &'db dyn Db) -> NewTypeBase<'db> {
match self.eager_base(db) {
Some(base) => base,
None => self.lazy_base(db),
}
}
#[salsa::tracked(
cycle_initial=lazy_base_cycle_initial,
heap_size=ruff_memory_usage::heap_size
)]
fn lazy_base(self, db: &'db dyn Db) -> NewTypeBase<'db> {
// `TypeInferenceBuilder` emits diagnostics for invalid `NewType` definitions that show up
// in assignments, but invalid definitions still get here, and also `NewType` might show up
// in places that aren't definitions at all. Fall back to `object` in all error cases.
let object_fallback = NewTypeBase::ClassType(ClassType::object(db));
let definition = self.definition(db);
let module = parsed_module(db, definition.file(db)).load(db);
let DefinitionKind::Assignment(assignment) = definition.kind(db) else {
return object_fallback;
};
let Some(call_expr) = assignment.value(&module).as_call_expr() else {
return object_fallback;
};
let Some(second_arg) = call_expr.arguments.args.get(1) else {
return object_fallback;
};
match definition_expression_type(db, definition, second_arg) {
Type::NominalInstance(nominal_instance_type) => {
NewTypeBase::ClassType(nominal_instance_type.class(db))
}
Type::NewTypeInstance(newtype) => NewTypeBase::NewType(newtype),
// This branch includes bases that are other typing constructs besides classes and
// other newtypes, for example unions. `NewType("Foo", int | str)` is not allowed.
_ => object_fallback,
}
}
fn iter_bases(self, db: &'db dyn Db) -> NewTypeBaseIter<'db> {
NewTypeBaseIter {
current: Some(self),
seen_before: BTreeSet::new(),
db,
}
}
// Walk the `NewTypeBase` chain to find the underlying `ClassType`. There might not be a
// `ClassType` if this `NewType` is cyclical, and we fall back to `object` in that case.
pub fn base_class_type(self, db: &'db dyn Db) -> ClassType<'db> {
for base in self.iter_bases(db) {
if let NewTypeBase::ClassType(class_type) = base {
return class_type;
}
}
ClassType::object(db)
}
pub(crate) fn is_equivalent_to_impl(self, db: &'db dyn Db, other: Self) -> bool {
// Two instances of the "same" `NewType` won't compare == if one of them has an eagerly
// evaluated base (or a normalized base, etc.) and the other doesn't, so we only check for
// equality of the `definition`.
self.definition(db) == other.definition(db)
}
// Since a regular class can't inherit from a newtype, the only way for one newtype to be a
// subtype of another is to have the other in its chain of newtype bases. Once we reach the
// base class, we don't have to keep looking.
pub(crate) fn has_relation_to_impl(self, db: &'db dyn Db, other: Self) -> ConstraintSet<'db> {
if self.is_equivalent_to_impl(db, other) {
return ConstraintSet::from(true);
}
for base in self.iter_bases(db) {
if let NewTypeBase::NewType(base_newtype) = base {
if base_newtype.is_equivalent_to_impl(db, other) {
return ConstraintSet::from(true);
}
}
}
ConstraintSet::from(false)
}
pub(crate) fn is_disjoint_from_impl(self, db: &'db dyn Db, other: Self) -> ConstraintSet<'db> {
// Two NewTypes are disjoint if they're not equal and neither inherits from the other.
// NewTypes have single inheritance, and a regular class can't inherit from a NewType, so
// it's not possible for some third type to multiply-inherit from both.
let mut self_not_subtype_of_other = self.has_relation_to_impl(db, other).negate(db);
let other_not_subtype_of_self = other.has_relation_to_impl(db, self).negate(db);
self_not_subtype_of_other.intersect(db, other_not_subtype_of_self)
}
/// Create a new `NewType` by mapping the underlying `ClassType`. This descends through any
/// number of nested `NewType` layers and rebuilds the whole chain. In the rare case of cyclic
/// `NewType`s with no underlying `ClassType`, this has no effect and does not call `f`.
pub(crate) fn map_base_class_type(
self,
db: &'db dyn Db,
f: impl FnOnce(ClassType<'db>) -> ClassType<'db>,
) -> Self {
// Modifying the base class type requires unwrapping and re-wrapping however many base
// newtypes there are between here and there. Normally recursion would be natural for this,
// but the bases iterator does cycle detection, and I think using that with a stack is a
// little cleaner than conjuring up yet another `CycleDetector` visitor and yet another
// layer of "*_impl" nesting. Also if there is no base class type, returning `self`
// unmodified seems more correct than injecting some default type like `object` into the
// cycle, which is what `CycleDetector` would do if we used it here.
let mut inner_newtype_stack = Vec::new();
for base in self.iter_bases(db) {
match base {
// Build up the stack of intermediate newtypes that we'll need to re-wrap after
// we've mapped the `ClassType`.
NewTypeBase::NewType(base_newtype) => inner_newtype_stack.push(base_newtype),
// We've reached the `ClassType`.
NewTypeBase::ClassType(base_class_type) => {
// Call `f`.
let mut mapped_base = NewTypeBase::ClassType(f(base_class_type));
// Re-wrap the mapped base class in however many newtypes we unwrapped.
for inner_newtype in inner_newtype_stack.into_iter().rev() {
mapped_base = NewTypeBase::NewType(NewType::new(
db,
inner_newtype.name(db).clone(),
inner_newtype.definition(db),
Some(mapped_base),
));
}
return NewType::new(
db,
self.name(db).clone(),
self.definition(db),
Some(mapped_base),
);
}
}
}
// If we get here, there is no `ClassType` (because this newtype is cyclic), and we don't
// call `f` at all.
self
}
}
pub(crate) fn walk_newtype_instance_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>(
db: &'db dyn Db,
newtype: NewType<'db>,
visitor: &V,
) {
visitor.visit_type(db, newtype.base(db).instance_type(db));
}
/// `typing.NewType` typically wraps a class type, but it can also wrap another newtype.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, get_size2::GetSize, salsa::Update)]
pub enum NewTypeBase<'db> {
ClassType(ClassType<'db>),
NewType(NewType<'db>),
}
impl<'db> NewTypeBase<'db> {
pub fn instance_type(self, db: &'db dyn Db) -> Type<'db> {
match self {
NewTypeBase::ClassType(class_type) => Type::instance(db, class_type),
NewTypeBase::NewType(newtype) => Type::NewTypeInstance(newtype),
}
}
}
/// An iterator over the transitive bases of a `NewType`. In the most common case, e.g.
/// `Foo = NewType("Foo", int)`, this yields the one `NewTypeBase::ClassType` (e.g. `int`). For
/// newtypes that wrap other newtypes, this iterator yields the `NewTypeBase::NewType`s (not
/// including `self`) before finally yielding the `NewTypeBase::ClassType`. In the pathological
/// case of cyclic newtypes like `Foo = NewType("Foo", "Foo")`, this iterator yields the unique
/// `NewTypeBase::NewType`s (not including `self`), detects the cycle, and then stops.
///
/// Note that this does *not* detect indirect cycles that go through a proper class, like this:
/// ```py
/// Foo = NewType("Foo", list["Foo"])
/// ```
/// As far as this iterator is concerned, that's the "common case", and it yields the one
/// `NewTypeBase::ClassType` for `list[Foo]`. Functions like `normalize` that continue recursing
/// over the base class need to pass down a cycle-detecting visitor as usual.
struct NewTypeBaseIter<'db> {
current: Option<NewType<'db>>,
seen_before: BTreeSet<NewType<'db>>,
db: &'db dyn Db,
}
impl<'db> Iterator for NewTypeBaseIter<'db> {
type Item = NewTypeBase<'db>;
fn next(&mut self) -> Option<Self::Item> {
let current = self.current?;
match current.base(self.db) {
NewTypeBase::ClassType(base_class_type) => {
self.current = None;
Some(NewTypeBase::ClassType(base_class_type))
}
NewTypeBase::NewType(base_newtype) => {
// Doing the insertion only in this branch avoids allocating in the common case.
self.seen_before.insert(current);
if self.seen_before.contains(&base_newtype) {
// Cycle detected. Stop iterating.
self.current = None;
None
} else {
self.current = Some(base_newtype);
Some(NewTypeBase::NewType(base_newtype))
}
}
}
}
}
fn lazy_base_cycle_initial<'db>(
db: &'db dyn Db,
_id: salsa::Id,
_self: NewType<'db>,
) -> NewTypeBase<'db> {
NewTypeBase::ClassType(ClassType::object(db))
}

4
crates/ty_python_semantic/src/types/type_ordering.rs
View File

@@ -213,6 +213,10 @@ pub(super) fn union_or_intersection_elements_ordering<'db>(
(Type::TypedDict(_), _) => Ordering::Less,
(_, Type::TypedDict(_)) => Ordering::Greater,
(Type::NewTypeInstance(left), Type::NewTypeInstance(right)) => left.cmp(right),
(Type::NewTypeInstance(_), _) => Ordering::Less,
(_, Type::NewTypeInstance(_)) => Ordering::Greater,
(Type::Union(_), _) | (_, Type::Union(_)) => {
unreachable!("our type representation does not permit nested unions");
}

12
crates/ty_python_semantic/src/types/visitor.rs
View File

@@ -11,6 +11,7 @@ use crate::{
class::walk_generic_alias,
function::{FunctionType, walk_function_type},
instance::{walk_nominal_instance_type, walk_protocol_instance_type},
newtype::{NewType, walk_newtype_instance_type},
subclass_of::walk_subclass_of_type,
walk_bound_method_type, walk_bound_type_var_type, walk_callable_type,
walk_intersection_type, walk_known_instance_type, walk_method_wrapper_type,
@@ -109,6 +110,10 @@ pub(crate) trait TypeVisitor<'db> {
fn visit_typed_dict_type(&self, db: &'db dyn Db, typed_dict: TypedDictType<'db>) {
walk_typed_dict_type(db, typed_dict, self);
}
fn visit_newtype_instance_type(&self, db: &'db dyn Db, newtype: NewType<'db>) {
walk_newtype_instance_type(db, newtype, self);
}
}
/// Enumeration of types that may contain other types, such as unions, intersections, and generics.
@@ -131,6 +136,7 @@ pub(super) enum NonAtomicType<'db> {
ProtocolInstance(ProtocolInstanceType<'db>),
TypedDict(TypedDictType<'db>),
TypeAlias(TypeAliasType<'db>),
NewTypeInstance(NewType<'db>),
}
pub(super) enum TypeKind<'db> {
@@ -198,6 +204,9 @@ impl<'db> From<Type<'db>> for TypeKind<'db> {
TypeKind::NonAtomic(NonAtomicType::TypedDict(typed_dict))
}
Type::TypeAlias(alias) => TypeKind::NonAtomic(NonAtomicType::TypeAlias(alias)),
Type::NewTypeInstance(newtype) => {
TypeKind::NonAtomic(NonAtomicType::NewTypeInstance(newtype))
}
}
}
}
@@ -239,6 +248,9 @@ pub(super) fn walk_non_atomic_type<'db, V: TypeVisitor<'db> + ?Sized>(
NonAtomicType::TypeAlias(alias) => {
visitor.visit_type_alias_type(db, alias);
}
NonAtomicType::NewTypeInstance(newtype) => {
visitor.visit_newtype_instance_type(db, newtype);
}
}
}