From b08f0b2caaf309d875099349b6f695d59d9753ea Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 3 Dec 2025 15:49:51 +0000 Subject: [PATCH 01/41] [ty] Sync vendored typeshed stubs (#21715) Co-authored-by: typeshedbot <> Co-authored-by: David Peter --- crates/ty_ide/src/goto_type_definition.rs | 12 ++++----- ...e_`@depr…_-_Syntax_(142fa2948c3c6cf1).snap | 12 ++++----- .../vendor/typeshed/source_commit.txt | 2 +- .../typeshed/stdlib/asyncio/runners.pyi | 11 +++++--- .../vendor/typeshed/stdlib/asyncio/trsock.pyi | 2 +- .../vendor/typeshed/stdlib/calendar.pyi | 4 ++- .../vendor/typeshed/stdlib/ipaddress.pyi | 2 +- .../vendor/typeshed/stdlib/mimetypes.pyi | 12 ++++----- .../stdlib/multiprocessing/managers.pyi | 5 +++- .../stdlib/multiprocessing/process.pyi | 6 +++++ .../stdlib/multiprocessing/synchronize.pyi | 3 +++ .../typeshed/stdlib/sqlite3/__init__.pyi | 5 ++-- .../vendor/typeshed/stdlib/subprocess.pyi | 1 - .../vendor/typeshed/stdlib/sys/__init__.pyi | 14 +++++++---- .../vendor/typeshed/stdlib/threading.pyi | 3 +++ .../typeshed/stdlib/tkinter/constants.pyi | 12 ++++----- .../vendor/typeshed/stdlib/typing.pyi | 25 +++++++++++++------ .../typeshed/stdlib/typing_extensions.pyi | 1 + .../vendor/typeshed/stdlib/unittest/mock.pyi | 1 + .../vendor/typeshed/stdlib/urllib/request.pyi | 7 ++++++ .../typeshed/stdlib/xml/etree/ElementTree.pyi | 8 ++++++ .../typeshed/stdlib/zipfile/__init__.pyi | 9 +++++++ 22 files changed, 108 insertions(+), 49 deletions(-) diff --git a/crates/ty_ide/src/goto_type_definition.rs b/crates/ty_ide/src/goto_type_definition.rs index fe85f44095..5964f241a4 100644 --- a/crates/ty_ide/src/goto_type_definition.rs +++ b/crates/ty_ide/src/goto_type_definition.rs @@ -145,14 +145,14 @@ mod tests { assert_snapshot!(test.goto_type_definition(), @r" info[goto-type-definition]: Type definition - --> stdlib/typing.pyi:770:1 + --> stdlib/typing.pyi:781:1 | - 768 | def __class_getitem__(cls, args: TypeVar | tuple[TypeVar, ...]) -> _Final: ... - 769 | - 770 | Generic: type[_Generic] + 779 | def __class_getitem__(cls, args: TypeVar | tuple[TypeVar, ...]) -> _Final: ... + 780 | + 781 | Generic: type[_Generic] | ^^^^^^^ - 771 | - 772 | class _ProtocolMeta(ABCMeta): + 782 | + 783 | class _ProtocolMeta(ABCMeta): | info: Source --> main.py:4:1 diff --git a/crates/ty_python_semantic/resources/mdtest/snapshots/deprecated.md_-_Tests_for_the_`@depr…_-_Syntax_(142fa2948c3c6cf1).snap b/crates/ty_python_semantic/resources/mdtest/snapshots/deprecated.md_-_Tests_for_the_`@depr…_-_Syntax_(142fa2948c3c6cf1).snap index 78cbed24b6..d8bf66cf9e 100644 --- a/crates/ty_python_semantic/resources/mdtest/snapshots/deprecated.md_-_Tests_for_the_`@depr…_-_Syntax_(142fa2948c3c6cf1).snap +++ b/crates/ty_python_semantic/resources/mdtest/snapshots/deprecated.md_-_Tests_for_the_`@depr…_-_Syntax_(142fa2948c3c6cf1).snap @@ -91,14 +91,14 @@ error[missing-argument]: No argument provided for required parameter `arg` of bo 7 | from typing_extensions import deprecated | info: Parameter declared here - --> stdlib/typing_extensions.pyi:1000:28 + --> stdlib/typing_extensions.pyi:1001:28 | - 998 | stacklevel: int - 999 | def __init__(self, message: LiteralString, /, *, category: type[Warning] | None = ..., stacklevel: int = 1) -> None: ... -1000 | def __call__(self, arg: _T, /) -> _T: ... + 999 | stacklevel: int +1000 | def __init__(self, message: LiteralString, /, *, category: type[Warning] | None = ..., stacklevel: int = 1) -> None: ... +1001 | def __call__(self, arg: _T, /) -> _T: ... | ^^^^^^^ -1001 | -1002 | @final +1002 | +1003 | @final | info: rule `missing-argument` is enabled by default diff --git a/crates/ty_vendored/vendor/typeshed/source_commit.txt b/crates/ty_vendored/vendor/typeshed/source_commit.txt index 7ce1784405..fae00078a9 100644 --- a/crates/ty_vendored/vendor/typeshed/source_commit.txt +++ b/crates/ty_vendored/vendor/typeshed/source_commit.txt @@ -1 +1 @@ -f8cdc0bd526301e873cd952eb0d457bdf2554e57 +ef2b90c67e5c668b91b3ae121baf00ee5165c30b diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/asyncio/runners.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/asyncio/runners.pyi index 25698e14a6..efc5eaac66 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/asyncio/runners.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/asyncio/runners.pyi @@ -1,6 +1,6 @@ import sys from _typeshed import Unused -from collections.abc import Callable, Coroutine +from collections.abc import Awaitable, Callable, Coroutine from contextvars import Context from typing import Any, TypeVar, final from typing_extensions import Self @@ -50,9 +50,12 @@ if sys.version_info >= (3, 11): def get_loop(self) -> AbstractEventLoop: """Return embedded event loop.""" - - def run(self, coro: Coroutine[Any, Any, _T], *, context: Context | None = None) -> _T: - """Run code in the embedded event loop.""" + if sys.version_info >= (3, 14): + def run(self, coro: Awaitable[_T], *, context: Context | None = None) -> _T: + """Run code in the embedded event loop.""" + else: + def run(self, coro: Coroutine[Any, Any, _T], *, context: Context | None = None) -> _T: + """Run a coroutine inside the embedded event loop.""" if sys.version_info >= (3, 12): def run( diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/asyncio/trsock.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/asyncio/trsock.pyi index 4d08d24016..ebcd409057 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/asyncio/trsock.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/asyncio/trsock.pyi @@ -69,7 +69,7 @@ class TransportSocket: def listen(self, backlog: int = ..., /) -> None: ... @deprecated("Removed in Python 3.11") def makefile(self) -> BinaryIO: ... - @deprecated("Rmoved in Python 3.11") + @deprecated("Removed in Python 3.11") def sendfile(self, file: BinaryIO, offset: int = 0, count: int | None = None) -> int: ... @deprecated("Removed in Python 3.11") def close(self) -> None: ... diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/calendar.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/calendar.pyi index 3b2aa61ceb..b0cda899d6 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/calendar.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/calendar.pyi @@ -64,10 +64,12 @@ if sys.version_info >= (3, 12): _LocaleType: TypeAlias = tuple[str | None, str | None] -class IllegalMonthError(ValueError): +class IllegalMonthError(ValueError, IndexError): + month: int def __init__(self, month: int) -> None: ... class IllegalWeekdayError(ValueError): + weekday: int def __init__(self, weekday: int) -> None: ... def isleap(year: int) -> bool: diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/ipaddress.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/ipaddress.pyi index f8397ff5b3..eb6439f5ad 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/ipaddress.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/ipaddress.pyi @@ -257,7 +257,7 @@ class _BaseNetwork(_IPAddressBase, Generic[_A]): """ - def hosts(self) -> Iterator[_A] | list[_A]: + def hosts(self) -> Iterator[_A]: """Generate Iterator over usable hosts in a network. This is like __iter__ except it doesn't return the network diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/mimetypes.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/mimetypes.pyi index 4e54a0eb85..0df876c5aa 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/mimetypes.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/mimetypes.pyi @@ -25,7 +25,7 @@ read_mime_types(file) -- parse one file, return a dictionary or None import sys from _typeshed import StrPath -from collections.abc import Sequence +from collections.abc import Iterable from typing import IO __all__ = [ @@ -93,8 +93,8 @@ def guess_extension(type: str, strict: bool = True) -> str | None: but non-standard types. """ -def init(files: Sequence[str] | None = None) -> None: ... -def read_mime_types(file: str) -> dict[str, str] | None: ... +def init(files: Iterable[StrPath] | None = None) -> None: ... +def read_mime_types(file: StrPath) -> dict[str, str] | None: ... def add_type(type: str, ext: str, strict: bool = True) -> None: """Add a mapping between a type and an extension. @@ -116,7 +116,7 @@ if sys.version_info >= (3, 13): """ inited: bool -knownfiles: list[str] +knownfiles: list[StrPath] suffix_map: dict[str, str] encodings_map: dict[str, str] types_map: dict[str, str] @@ -134,7 +134,7 @@ class MimeTypes: encodings_map: dict[str, str] types_map: tuple[dict[str, str], dict[str, str]] types_map_inv: tuple[dict[str, str], dict[str, str]] - def __init__(self, filenames: tuple[str, ...] = (), strict: bool = True) -> None: ... + def __init__(self, filenames: Iterable[StrPath] = (), strict: bool = True) -> None: ... def add_type(self, type: str, ext: str, strict: bool = True) -> None: """Add a mapping between a type and an extension. @@ -196,7 +196,7 @@ class MimeTypes: but non-standard types. """ - def read(self, filename: str, strict: bool = True) -> None: + def read(self, filename: StrPath, strict: bool = True) -> None: """ Read a single mime.types-format file, specified by pathname. diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/multiprocessing/managers.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/multiprocessing/managers.pyi index c4c8182c1a..082267fdad 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/multiprocessing/managers.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/multiprocessing/managers.pyi @@ -216,6 +216,8 @@ class BaseListProxy(BaseProxy, MutableSequence[_T]): def count(self, value: _T, /) -> int: ... def insert(self, index: SupportsIndex, object: _T, /) -> None: ... def remove(self, value: _T, /) -> None: ... + if sys.version_info >= (3, 14): + def copy(self) -> list[_T]: ... # Use BaseListProxy[SupportsRichComparisonT] for the first overload rather than [SupportsRichComparison] # to work around invariance @overload @@ -429,8 +431,9 @@ class SyncManager(BaseManager): def dict(self, iterable: Iterable[list[str]], /) -> DictProxy[str, str]: ... @overload def dict(self, iterable: Iterable[list[bytes]], /) -> DictProxy[bytes, bytes]: ... + # Overloads are copied from builtins.list.__init__ @overload - def list(self, sequence: Sequence[_T], /) -> ListProxy[_T]: ... + def list(self, iterable: Iterable[_T], /) -> ListProxy[_T]: ... @overload def list(self) -> ListProxy[Any]: ... if sys.version_info >= (3, 14): diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/multiprocessing/process.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/multiprocessing/process.pyi index f740eb50c0..26307a7fe3 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/multiprocessing/process.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/multiprocessing/process.pyi @@ -1,3 +1,4 @@ +import sys from collections.abc import Callable, Iterable, Mapping from typing import Any @@ -33,6 +34,11 @@ class BaseProcess: """ Start child process """ + if sys.version_info >= (3, 14): + def interrupt(self) -> None: + """ + Terminate process; sends SIGINT signal + """ def terminate(self) -> None: """ diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/multiprocessing/synchronize.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/multiprocessing/synchronize.pyi index a0d97baa06..541e0b05dd 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/multiprocessing/synchronize.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/multiprocessing/synchronize.pyi @@ -1,3 +1,4 @@ +import sys import threading from collections.abc import Callable from multiprocessing.context import BaseContext @@ -45,6 +46,8 @@ class SemLock: # These methods are copied from the wrapped _multiprocessing.SemLock object def acquire(self, block: bool = True, timeout: float | None = None) -> bool: ... def release(self) -> None: ... + if sys.version_info >= (3, 14): + def locked(self) -> bool: ... class Lock(SemLock): def __init__(self, *, ctx: BaseContext) -> None: ... diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/sqlite3/__init__.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/sqlite3/__init__.pyi index e378b4d434..b57c0410dc 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/sqlite3/__init__.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/sqlite3/__init__.pyi @@ -256,6 +256,7 @@ _AdaptedInputData: TypeAlias = _SqliteData | Any _Parameters: TypeAlias = SupportsLenAndGetItem[_AdaptedInputData] | Mapping[str, _AdaptedInputData] # Controls the legacy transaction handling mode of sqlite3. _IsolationLevel: TypeAlias = Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None +_RowFactoryOptions: TypeAlias = type[Row] | Callable[[Cursor, Row], object] | None @type_check_only class _AnyParamWindowAggregateClass(Protocol): @@ -336,7 +337,7 @@ class Connection: def autocommit(self) -> int: ... @autocommit.setter def autocommit(self, val: int) -> None: ... - row_factory: Any + row_factory: _RowFactoryOptions text_factory: Any if sys.version_info >= (3, 12): def __init__( @@ -623,7 +624,7 @@ class Cursor: def description(self) -> tuple[tuple[str, None, None, None, None, None, None], ...] | MaybeNone: ... @property def lastrowid(self) -> int | None: ... - row_factory: Callable[[Cursor, Row], object] | None + row_factory: _RowFactoryOptions @property def rowcount(self) -> int: ... def __init__(self, cursor: Connection, /) -> None: ... diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/subprocess.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/subprocess.pyi index 60e4906577..090c41209d 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/subprocess.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/subprocess.pyi @@ -606,7 +606,6 @@ elif sys.version_info >= (3, 10): ) -> CompletedProcess[Any]: ... else: - # 3.9 adds arguments "user", "group", "extra_groups" and "umask" @overload def run( args: _CMD, diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/sys/__init__.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/sys/__init__.pyi index eef3dd37b5..ee3a99be81 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/sys/__init__.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/sys/__init__.pyi @@ -77,7 +77,7 @@ from builtins import object as _object from collections.abc import AsyncGenerator, Callable, Sequence from io import TextIOWrapper from types import FrameType, ModuleType, TracebackType -from typing import Any, Final, Literal, NoReturn, Protocol, TextIO, TypeVar, final, type_check_only +from typing import Any, Final, Literal, NoReturn, Protocol, TextIO, TypeVar, final, overload, type_check_only from typing_extensions import LiteralString, TypeAlias, deprecated _T = TypeVar("_T") @@ -648,7 +648,7 @@ if sys.platform == "android": # noqa: Y008 def getallocatedblocks() -> int: """Return the number of memory blocks currently allocated.""" -def getdefaultencoding() -> str: +def getdefaultencoding() -> Literal["utf-8"]: """Return the current default encoding used by the Unicode implementation.""" if sys.platform != "win32": @@ -658,10 +658,10 @@ if sys.platform != "win32": The flag constants are defined in the os module. """ -def getfilesystemencoding() -> str: +def getfilesystemencoding() -> LiteralString: """Return the encoding used to convert Unicode filenames to OS filenames.""" -def getfilesystemencodeerrors() -> str: +def getfilesystemencodeerrors() -> LiteralString: """Return the error mode used Unicode to OS filename conversion.""" def getrefcount(object: Any, /) -> int: @@ -755,7 +755,8 @@ if sys.platform == "win32": intended for identifying the OS rather than feature detection. """ -def intern(string: str, /) -> str: +@overload +def intern(string: LiteralString, /) -> LiteralString: """``Intern'' the given string. This enters the string in the (global) table of interned strings whose @@ -763,6 +764,9 @@ def intern(string: str, /) -> str: the previously interned string object with the same value. """ +@overload +def intern(string: str, /) -> str: ... # type: ignore[misc] + __interactivehook__: Callable[[], object] if sys.version_info >= (3, 13): diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/threading.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/threading.pyi index 4a52cc0561..d043edc4a3 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/threading.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/threading.pyi @@ -447,6 +447,9 @@ class Condition: ) -> None: ... def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: ... def release(self) -> None: ... + if sys.version_info >= (3, 14): + def locked(self) -> bool: ... + def wait(self, timeout: float | None = None) -> bool: """Wait until notified or until a timeout occurs. diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/tkinter/constants.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/tkinter/constants.pyi index fbfe8b49b9..eb1ef446cf 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/tkinter/constants.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/tkinter/constants.pyi @@ -1,12 +1,12 @@ from typing import Final # These are not actually bools. See #4669 -NO: Final[bool] -YES: Final[bool] -TRUE: Final[bool] -FALSE: Final[bool] -ON: Final[bool] -OFF: Final[bool] +YES: Final = True +NO: Final = False +TRUE: Final = True +FALSE: Final = False +ON: Final = True +OFF: Final = False N: Final = "n" S: Final = "s" W: Final = "w" diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/typing.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/typing.pyi index 8daf975d2b..4f7606bc9b 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/typing.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/typing.pyi @@ -644,6 +644,7 @@ if sys.version_info >= (3, 10): def __or__(self, other: Any) -> _SpecialForm: ... def __ror__(self, other: Any) -> _SpecialForm: ... __supertype__: type | NewType + __name__: str else: def NewType(name: str, tp: Any) -> Any: @@ -722,12 +723,22 @@ def no_type_check(arg: _F) -> _F: This mutates the function(s) or class(es) in place. """ -def no_type_check_decorator(decorator: Callable[_P, _T]) -> Callable[_P, _T]: - """Decorator to give another decorator the @no_type_check effect. +if sys.version_info >= (3, 13): + @deprecated("Deprecated since Python 3.13; removed in Python 3.15.") + def no_type_check_decorator(decorator: Callable[_P, _T]) -> Callable[_P, _T]: + """Decorator to give another decorator the @no_type_check effect. - This wraps the decorator with something that wraps the decorated - function in @no_type_check. - """ + This wraps the decorator with something that wraps the decorated + function in @no_type_check. + """ + +else: + def no_type_check_decorator(decorator: Callable[_P, _T]) -> Callable[_P, _T]: + """Decorator to give another decorator the @no_type_check effect. + + This wraps the decorator with something that wraps the decorated + function in @no_type_check. + """ # This itself is only available during type checking def type_check_only(func_or_cls: _FT) -> _FT: ... @@ -1784,9 +1795,7 @@ class NamedTuple(tuple[Any, ...]): @overload def __init__(self, typename: str, fields: Iterable[tuple[str, Any]], /) -> None: ... @overload - @typing_extensions.deprecated( - "Creating a typing.NamedTuple using keyword arguments is deprecated and support will be removed in Python 3.15" - ) + @deprecated("Creating a typing.NamedTuple using keyword arguments is deprecated and support will be removed in Python 3.15") def __init__(self, typename: str, fields: None = None, /, **kwargs: Any) -> None: ... @classmethod def _make(cls, iterable: Iterable[Any]) -> typing_extensions.Self: ... diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/typing_extensions.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/typing_extensions.pyi index 1e81194ead..2c42633cf9 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/typing_extensions.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/typing_extensions.pyi @@ -702,6 +702,7 @@ else: def __init__(self, name: str, tp: AnnotationForm) -> None: ... def __call__(self, obj: _T, /) -> _T: ... __supertype__: type | NewType + __name__: str if sys.version_info >= (3, 10): def __or__(self, other: Any) -> _SpecialForm: ... def __ror__(self, other: Any) -> _SpecialForm: ... diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/unittest/mock.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/unittest/mock.pyi index 615eec5fc4..bd8f519cc3 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/unittest/mock.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/unittest/mock.pyi @@ -322,6 +322,7 @@ class NonCallableMock(Base, Any): call_count: int call_args: _Call | MaybeNone call_args_list: _CallList + method_calls: _CallList mock_calls: _CallList def _format_mock_call_signature(self, args: Any, kwargs: Any) -> str: ... def _call_matcher(self, _call: tuple[_Call, ...]) -> _Call: diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/urllib/request.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/urllib/request.pyi index 4d9636102e..7c56838c49 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/urllib/request.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/urllib/request.pyi @@ -118,7 +118,14 @@ if sys.version_info < (3, 14): __all__ += ["URLopener", "FancyURLopener"] _T = TypeVar("_T") + +# The actual type is `addinfourl | HTTPResponse`, but users would need to use `typing.cast` or `isinstance` to narrow the type, +# so we use `Any` instead. +# See +# - https://github.com/python/typeshed/pull/15042 +# - https://github.com/python/typing/issues/566 _UrlopenRet: TypeAlias = Any + _DataType: TypeAlias = ReadableBuffer | SupportsRead[bytes] | Iterable[bytes] | None if sys.version_info >= (3, 13): diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/xml/etree/ElementTree.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/xml/etree/ElementTree.pyi index 99c3f287b6..5af84b4915 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/xml/etree/ElementTree.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/xml/etree/ElementTree.pyi @@ -260,6 +260,14 @@ class ElementTree(Generic[_Root]): def getroot(self) -> _Root: """Return root element of this tree.""" + def _setroot(self, element: Element[Any]) -> None: + """Replace root element of this tree. + + This will discard the current contents of the tree and replace it + with the given element. Use with care! + + """ + def parse(self, source: _FileRead, parser: XMLParser | None = None) -> Element: """Load external XML document into element tree. diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/zipfile/__init__.pyi b/crates/ty_vendored/vendor/typeshed/stdlib/zipfile/__init__.pyi index 0389fe1cba..822fcc81d2 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/zipfile/__init__.pyi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/zipfile/__init__.pyi @@ -483,6 +483,15 @@ class ZipInfo: decide based upon the file_size and compress_size, if known, False otherwise. """ + if sys.version_info >= (3, 14): + def _for_archive(self, archive: ZipFile) -> Self: + """Resolve suitable defaults from the archive. + + Resolve the date_time, compression attributes, and external attributes + to suitable defaults as used by :method:`ZipFile.writestr`. + + Return self. + """ if sys.version_info >= (3, 12): from zipfile._path import CompleteDirs as CompleteDirs, Path as Path From 4488e9d47de245c48403925d10655f7c71f42ee2 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 3 Dec 2025 11:07:29 -0500 Subject: [PATCH 02/41] Revert "Enable PEP 740 attestations when publishing to PyPI" (#21768) --- .github/workflows/publish-pypi.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index e2d1fe3587..a66345429a 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -18,7 +18,8 @@ jobs: environment: name: release permissions: - id-token: write # For PyPI's trusted publishing + PEP 740 attestations + # For PyPI's trusted publishing. + id-token: write steps: - name: "Install uv" uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4 @@ -27,8 +28,5 @@ jobs: pattern: wheels-* path: wheels merge-multiple: true - - uses: astral-sh/attest-action@2c727738cea36d6c97dd85eb133ea0e0e8fe754b # v0.0.4 - with: - paths: wheels/* - name: Publish to PyPi run: uv publish -v wheels/* From 1f4f8d9950fb433fd68337a40c40729af46376fa Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 3 Dec 2025 17:52:31 +0100 Subject: [PATCH 03/41] [ty] Fix flow of associated member states during star imports (#21776) ## Summary Star-imports can not just affect the state of symbols that they pull in, they can also affect the state of members that are associated with those symbols. For example, if `obj.attr` was previously narrowed from `int | None` to `int`, and a star-import now overwrites `obj`, then the narrowing on `obj.attr` should be "reset". This PR keeps track of the state of associated members during star imports and properly models the flow of their corresponding state through the control flow structure that we artificially create for star-imports. See [this comment](https://github.com/astral-sh/ty/issues/1355#issuecomment-3607125005) for an explanation why this caused ty to see certain `asyncio` symbols as not being accessible on Python 3.14. closes https://github.com/astral-sh/ty/issues/1355 ## Ecosystem impact ```diff async-utils (https://github.com/mikeshardmind/async-utils) - src/async_utils/bg_loop.py:115:31: error[invalid-argument-type] Argument to bound method `set_task_factory` is incorrect: Expected `_TaskFactory | None`, found `def eager_task_factory[_T_co](loop: AbstractEventLoop | None, coro: Coroutine[Any, Any, _T_co@eager_task_factory], *, name: str | None = None, context: Context | None = None) -> Task[_T_co@eager_task_factory]` - Found 30 diagnostics + Found 29 diagnostics mitmproxy (https://github.com/mitmproxy/mitmproxy) + mitmproxy/utils/asyncio_utils.py:96:60: warning[unused-ignore-comment] Unused blanket `type: ignore` directive - test/conftest.py:37:31: error[invalid-argument-type] Argument to bound method `set_task_factory` is incorrect: Expected `_TaskFactory | None`, found `def eager_task_factory[_T_co](loop: AbstractEventLoop | None, coro: Coroutine[Any, Any, _T_co@eager_task_factory], *, name: str | None = None, context: Context | None = None) -> Task[_T_co@eager_task_factory]` ``` All of these seem to be correct, they give us a different type for `asyncio` symbols that are now imported from different `sys.version_info` branches (where we previously failed to recognize some of these as statically true/false). ```diff dd-trace-py (https://github.com/DataDog/dd-trace-py) - ddtrace/contrib/internal/asyncio/patch.py:39:12: error[invalid-argument-type] Argument to function `unwrap` is incorrect: Expected `WrappedFunction`, found `def create_task[_T](self, coro: Coroutine[Any, Any, _T@create_task] | Generator[Any, None, _T@create_task], *, name: object = None) -> Task[_T@create_task]` + ddtrace/contrib/internal/asyncio/patch.py:39:12: error[invalid-argument-type] Argument to function `unwrap` is incorrect: Expected `WrappedFunction`, found `def create_task[_T](self, coro: Generator[Any, None, _T@create_task] | Coroutine[Any, Any, _T@create_task], *, name: object = None) -> Task[_T@create_task]` ``` Similar, but only results in a diagnostic change. ## Test Plan Added a regression test --- .../resources/mdtest/import/star.md | 63 +++++++++++++++++++ .../src/semantic_index/builder.rs | 5 +- .../src/semantic_index/use_def.rs | 54 ++++++++++++++-- 3 files changed, 116 insertions(+), 6 deletions(-) diff --git a/crates/ty_python_semantic/resources/mdtest/import/star.md b/crates/ty_python_semantic/resources/mdtest/import/star.md index 54d2050259..14cff45efc 100644 --- a/crates/ty_python_semantic/resources/mdtest/import/star.md +++ b/crates/ty_python_semantic/resources/mdtest/import/star.md @@ -1336,6 +1336,69 @@ reveal_type(g) # revealed: Unknown reveal_type(h) # revealed: Unknown ``` +## Star-imports can affect member states + +If a star-import pulls in a symbol that was previously defined in the importing module (e.g. `obj`), +it can affect the state of associated member expressions (e.g. `obj.attr` or `obj[0]`). In the test +below, note how the types of the corresponding attribute expressions change after the star import +affects the object: + +`common.py`: + +```py +class C: + attr: int | None +``` + +`exporter.py`: + +```py +from common import C + +def flag() -> bool: + return True + +should_be_imported: C = C() + +if flag(): + might_be_imported: C = C() + +if False: + should_not_be_imported: C = C() +``` + +`main.py`: + +```py +from common import C + +should_be_imported = C() +might_be_imported = C() +should_not_be_imported = C() + +# We start with the plain attribute types: +reveal_type(should_be_imported.attr) # revealed: int | None +reveal_type(might_be_imported.attr) # revealed: int | None +reveal_type(should_not_be_imported.attr) # revealed: int | None + +# Now we narrow the types by assignment: +should_be_imported.attr = 1 +might_be_imported.attr = 1 +should_not_be_imported.attr = 1 + +reveal_type(should_be_imported.attr) # revealed: Literal[1] +reveal_type(might_be_imported.attr) # revealed: Literal[1] +reveal_type(should_not_be_imported.attr) # revealed: Literal[1] + +# This star import adds bindings for `should_be_imported` and `might_be_imported`: +from exporter import * + +# As expected, narrowing is "reset" for the first two variables, but not for the third: +reveal_type(should_be_imported.attr) # revealed: int | None +reveal_type(might_be_imported.attr) # revealed: int | None +reveal_type(should_not_be_imported.attr) # revealed: Literal[1] +``` + ## Cyclic star imports Believe it or not, this code does *not* raise an exception at runtime! diff --git a/crates/ty_python_semantic/src/semantic_index/builder.rs b/crates/ty_python_semantic/src/semantic_index/builder.rs index 66a0f6f428..9ba2069784 100644 --- a/crates/ty_python_semantic/src/semantic_index/builder.rs +++ b/crates/ty_python_semantic/src/semantic_index/builder.rs @@ -1616,9 +1616,12 @@ impl<'ast> Visitor<'ast> for SemanticIndexBuilder<'_, 'ast> { let star_import_predicate = self.add_predicate(star_import.into()); + let associated_member_ids = self.place_tables[self.current_scope()] + .associated_place_ids(ScopedPlaceId::Symbol(symbol_id)); let pre_definition = self .current_use_def_map() - .single_symbol_place_snapshot(symbol_id); + .single_symbol_snapshot(symbol_id, associated_member_ids); + let pre_definition_reachability = self.current_use_def_map().reachability; diff --git a/crates/ty_python_semantic/src/semantic_index/use_def.rs b/crates/ty_python_semantic/src/semantic_index/use_def.rs index 05fa369521..a7c7520806 100644 --- a/crates/ty_python_semantic/src/semantic_index/use_def.rs +++ b/crates/ty_python_semantic/src/semantic_index/use_def.rs @@ -801,6 +801,13 @@ pub(super) struct FlowSnapshot { reachability: ScopedReachabilityConstraintId, } +/// A snapshot of the state of a single symbol (e.g. `obj`) and all of its associated members +/// (e.g. `obj.attr`, `obj["key"]`). +pub(super) struct SingleSymbolSnapshot { + symbol_state: PlaceState, + associated_member_states: FxHashMap, +} + #[derive(Debug)] pub(super) struct UseDefMapBuilder<'db> { /// Append-only array of [`DefinitionState`]. @@ -991,13 +998,26 @@ impl<'db> UseDefMapBuilder<'db> { } } - /// Snapshot the state of a single place at the current point in control flow. + /// Snapshot the state of a single symbol and all of its associated members, at the current + /// point in control flow. /// /// This is only used for `*`-import reachability constraints, which are handled differently /// to most other reachability constraints. See the doc-comment for /// [`Self::record_and_negate_star_import_reachability_constraint`] for more details. - pub(super) fn single_symbol_place_snapshot(&self, symbol: ScopedSymbolId) -> PlaceState { - self.symbol_states[symbol].clone() + pub(super) fn single_symbol_snapshot( + &self, + symbol: ScopedSymbolId, + associated_member_ids: &[ScopedMemberId], + ) -> SingleSymbolSnapshot { + let symbol_state = self.symbol_states[symbol].clone(); + let mut associated_member_states = FxHashMap::default(); + for &member_id in associated_member_ids { + associated_member_states.insert(member_id, self.member_states[member_id].clone()); + } + SingleSymbolSnapshot { + symbol_state, + associated_member_states, + } } /// This method exists solely for handling `*`-import reachability constraints. @@ -1033,14 +1053,14 @@ impl<'db> UseDefMapBuilder<'db> { &mut self, reachability_id: ScopedReachabilityConstraintId, symbol: ScopedSymbolId, - pre_definition_state: PlaceState, + pre_definition: SingleSymbolSnapshot, ) { let negated_reachability_id = self .reachability_constraints .add_not_constraint(reachability_id); let mut post_definition_state = - std::mem::replace(&mut self.symbol_states[symbol], pre_definition_state); + std::mem::replace(&mut self.symbol_states[symbol], pre_definition.symbol_state); post_definition_state .record_reachability_constraint(&mut self.reachability_constraints, reachability_id); @@ -1055,6 +1075,30 @@ impl<'db> UseDefMapBuilder<'db> { &mut self.narrowing_constraints, &mut self.reachability_constraints, ); + + // And similarly for all associated members: + for (member_id, pre_definition_member_state) in pre_definition.associated_member_states { + let mut post_definition_state = std::mem::replace( + &mut self.member_states[member_id], + pre_definition_member_state, + ); + + post_definition_state.record_reachability_constraint( + &mut self.reachability_constraints, + reachability_id, + ); + + self.member_states[member_id].record_reachability_constraint( + &mut self.reachability_constraints, + negated_reachability_id, + ); + + self.member_states[member_id].merge( + post_definition_state, + &mut self.narrowing_constraints, + &mut self.reachability_constraints, + ); + } } pub(super) fn record_reachability_constraint( From c722f498fe409f2897c746d0f40397a9460496e4 Mon Sep 17 00:00:00 2001 From: Bhuminjay Soni Date: Wed, 3 Dec 2025 22:35:15 +0530 Subject: [PATCH 04/41] [`flake8-bugbear`] Catch `yield` expressions within other statements (`B901`) (#21200) ## Summary This PR re-implements [return-in-generator (B901)](https://docs.astral.sh/ruff/rules/return-in-generator/#return-in-generator-b901) for async generators as a semantic syntax error. This is not a syntax error for sync generators, so we'll need to preserve both the lint rule and the syntax error in this case. It also updates B901 and the new implementation to catch cases where the generator's `yield` or `yield from` expression is part of another statement, as in: ```py def foo(): return (yield) ``` These were previously not caught because we only looked for `Stmt::Expr(Expr::Yield)` in `visit_stmt` instead of visiting `yield` expressions directly. I think this modification is within the spirit of the rule and safe to try out since the rule is in preview. ## Test Plan I have written tests as directed in #17412 --------- Signed-off-by: 11happy Signed-off-by: 11happy Co-authored-by: Brent Westbrook Co-authored-by: Brent Westbrook <36778786+ntBre@users.noreply.github.com> --- .../test/fixtures/flake8_bugbear/B901.py | 16 ++++- .../syntax_errors/return_in_generator.py | 24 ++++++++ crates/ruff_linter/src/checkers/ast/mod.rs | 7 +++ crates/ruff_linter/src/linter.rs | 1 + .../rules/return_in_generator.rs | 29 ++++++--- ...__flake8_bugbear__tests__B901_B901.py.snap | 43 +++++++++++++ ...linter__tests__return_in_generator.py.snap | 55 +++++++++++++++++ .../ruff_python_parser/src/semantic_errors.rs | 61 ++++++++++++++++++- 8 files changed, 220 insertions(+), 16 deletions(-) create mode 100644 crates/ruff_linter/resources/test/fixtures/syntax_errors/return_in_generator.py create mode 100644 crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__return_in_generator.py.snap diff --git a/crates/ruff_linter/resources/test/fixtures/flake8_bugbear/B901.py b/crates/ruff_linter/resources/test/fixtures/flake8_bugbear/B901.py index 42fdda60d7..acb932f25f 100644 --- a/crates/ruff_linter/resources/test/fixtures/flake8_bugbear/B901.py +++ b/crates/ruff_linter/resources/test/fixtures/flake8_bugbear/B901.py @@ -52,16 +52,16 @@ def not_broken5(): yield inner() -def not_broken6(): +def broken3(): return (yield from []) -def not_broken7(): +def broken4(): x = yield from [] return x -def not_broken8(): +def broken5(): x = None def inner(ex): @@ -76,3 +76,13 @@ class NotBroken9(object): def __await__(self): yield from function() return 42 + + +async def broken6(): + yield 1 + return foo() + + +async def broken7(): + yield 1 + return [1, 2, 3] diff --git a/crates/ruff_linter/resources/test/fixtures/syntax_errors/return_in_generator.py b/crates/ruff_linter/resources/test/fixtures/syntax_errors/return_in_generator.py new file mode 100644 index 0000000000..56ef47c17b --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/syntax_errors/return_in_generator.py @@ -0,0 +1,24 @@ +async def gen(): + yield 1 + return 42 + +def gen(): # B901 but not a syntax error - not an async generator + yield 1 + return 42 + +async def gen(): # ok - no value in return + yield 1 + return + +async def gen(): + yield 1 + return foo() + +async def gen(): + yield 1 + return [1, 2, 3] + +async def gen(): + if True: + yield 1 + return 10 diff --git a/crates/ruff_linter/src/checkers/ast/mod.rs b/crates/ruff_linter/src/checkers/ast/mod.rs index aa9fa839f2..4d4d7e9293 100644 --- a/crates/ruff_linter/src/checkers/ast/mod.rs +++ b/crates/ruff_linter/src/checkers/ast/mod.rs @@ -69,6 +69,7 @@ use crate::noqa::NoqaMapping; use crate::package::PackageRoot; use crate::preview::is_undefined_export_in_dunder_init_enabled; use crate::registry::Rule; +use crate::rules::flake8_bugbear::rules::ReturnInGenerator; use crate::rules::pyflakes::rules::{ LateFutureImport, MultipleStarredExpressions, ReturnOutsideFunction, UndefinedLocalWithNestedImportStarUsage, YieldOutsideFunction, @@ -729,6 +730,12 @@ impl SemanticSyntaxContext for Checker<'_> { self.report_diagnostic(NonlocalWithoutBinding { name }, error.range); } } + SemanticSyntaxErrorKind::ReturnInGenerator => { + // B901 + if self.is_rule_enabled(Rule::ReturnInGenerator) { + self.report_diagnostic(ReturnInGenerator, error.range); + } + } SemanticSyntaxErrorKind::ReboundComprehensionVariable | SemanticSyntaxErrorKind::DuplicateTypeParameter | SemanticSyntaxErrorKind::MultipleCaseAssignment(_) diff --git a/crates/ruff_linter/src/linter.rs b/crates/ruff_linter/src/linter.rs index 3ec070dd26..719d5ac9c5 100644 --- a/crates/ruff_linter/src/linter.rs +++ b/crates/ruff_linter/src/linter.rs @@ -1043,6 +1043,7 @@ mod tests { Rule::YieldFromInAsyncFunction, Path::new("yield_from_in_async_function.py") )] + #[test_case(Rule::ReturnInGenerator, Path::new("return_in_generator.py"))] fn test_syntax_errors(rule: Rule, path: &Path) -> Result<()> { let snapshot = path.to_string_lossy().to_string(); let path = Path::new("resources/test/fixtures/syntax_errors").join(path); diff --git a/crates/ruff_linter/src/rules/flake8_bugbear/rules/return_in_generator.rs b/crates/ruff_linter/src/rules/flake8_bugbear/rules/return_in_generator.rs index f7584dd4bb..0b089b3459 100644 --- a/crates/ruff_linter/src/rules/flake8_bugbear/rules/return_in_generator.rs +++ b/crates/ruff_linter/src/rules/flake8_bugbear/rules/return_in_generator.rs @@ -1,6 +1,5 @@ use ruff_macros::{ViolationMetadata, derive_message_formats}; -use ruff_python_ast::statement_visitor; -use ruff_python_ast::statement_visitor::StatementVisitor; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{self as ast, Expr, Stmt, StmtFunctionDef}; use ruff_text_size::TextRange; @@ -96,6 +95,11 @@ pub(crate) fn return_in_generator(checker: &Checker, function_def: &StmtFunction return; } + // Async functions are flagged by the `ReturnInGenerator` semantic syntax error. + if function_def.is_async { + return; + } + let mut visitor = ReturnInGeneratorVisitor::default(); visitor.visit_body(&function_def.body); @@ -112,15 +116,9 @@ struct ReturnInGeneratorVisitor { has_yield: bool, } -impl StatementVisitor<'_> for ReturnInGeneratorVisitor { +impl Visitor<'_> for ReturnInGeneratorVisitor { fn visit_stmt(&mut self, stmt: &Stmt) { match stmt { - Stmt::Expr(ast::StmtExpr { value, .. }) => match **value { - Expr::Yield(_) | Expr::YieldFrom(_) => { - self.has_yield = true; - } - _ => {} - }, Stmt::FunctionDef(_) => { // Do not recurse into nested functions; they're evaluated separately. } @@ -130,8 +128,19 @@ impl StatementVisitor<'_> for ReturnInGeneratorVisitor { node_index: _, }) => { self.return_ = Some(*range); + walk_stmt(self, stmt); } - _ => statement_visitor::walk_stmt(self, stmt), + _ => walk_stmt(self, stmt), + } + } + + fn visit_expr(&mut self, expr: &Expr) { + match expr { + Expr::Lambda(_) => {} + Expr::Yield(_) | Expr::YieldFrom(_) => { + self.has_yield = true; + } + _ => walk_expr(self, expr), } } } diff --git a/crates/ruff_linter/src/rules/flake8_bugbear/snapshots/ruff_linter__rules__flake8_bugbear__tests__B901_B901.py.snap b/crates/ruff_linter/src/rules/flake8_bugbear/snapshots/ruff_linter__rules__flake8_bugbear__tests__B901_B901.py.snap index 951860f81e..21bf1b1645 100644 --- a/crates/ruff_linter/src/rules/flake8_bugbear/snapshots/ruff_linter__rules__flake8_bugbear__tests__B901_B901.py.snap +++ b/crates/ruff_linter/src/rules/flake8_bugbear/snapshots/ruff_linter__rules__flake8_bugbear__tests__B901_B901.py.snap @@ -21,3 +21,46 @@ B901 Using `yield` and `return {value}` in a generator function can lead to conf 37 | 38 | yield from not_broken() | + +B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior + --> B901.py:56:5 + | +55 | def broken3(): +56 | return (yield from []) + | ^^^^^^^^^^^^^^^^^^^^^^ + | + +B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior + --> B901.py:61:5 + | +59 | def broken4(): +60 | x = yield from [] +61 | return x + | ^^^^^^^^ + | + +B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior + --> B901.py:72:5 + | +71 | inner((yield from [])) +72 | return x + | ^^^^^^^^ + | + +B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior + --> B901.py:83:5 + | +81 | async def broken6(): +82 | yield 1 +83 | return foo() + | ^^^^^^^^^^^^ + | + +B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior + --> B901.py:88:5 + | +86 | async def broken7(): +87 | yield 1 +88 | return [1, 2, 3] + | ^^^^^^^^^^^^^^^^ + | diff --git a/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__return_in_generator.py.snap b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__return_in_generator.py.snap new file mode 100644 index 0000000000..2abd1fad09 --- /dev/null +++ b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__return_in_generator.py.snap @@ -0,0 +1,55 @@ +--- +source: crates/ruff_linter/src/linter.rs +--- +B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior + --> resources/test/fixtures/syntax_errors/return_in_generator.py:3:5 + | +1 | async def gen(): +2 | yield 1 +3 | return 42 + | ^^^^^^^^^ +4 | +5 | def gen(): # B901 but not a syntax error - not an async generator + | + +B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior + --> resources/test/fixtures/syntax_errors/return_in_generator.py:7:5 + | +5 | def gen(): # B901 but not a syntax error - not an async generator +6 | yield 1 +7 | return 42 + | ^^^^^^^^^ +8 | +9 | async def gen(): # ok - no value in return + | + +B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior + --> resources/test/fixtures/syntax_errors/return_in_generator.py:15:5 + | +13 | async def gen(): +14 | yield 1 +15 | return foo() + | ^^^^^^^^^^^^ +16 | +17 | async def gen(): + | + +B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior + --> resources/test/fixtures/syntax_errors/return_in_generator.py:19:5 + | +17 | async def gen(): +18 | yield 1 +19 | return [1, 2, 3] + | ^^^^^^^^^^^^^^^^ +20 | +21 | async def gen(): + | + +B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior + --> resources/test/fixtures/syntax_errors/return_in_generator.py:24:5 + | +22 | if True: +23 | yield 1 +24 | return 10 + | ^^^^^^^^^ + | diff --git a/crates/ruff_python_parser/src/semantic_errors.rs b/crates/ruff_python_parser/src/semantic_errors.rs index 2c573271e1..0c7ceef4a4 100644 --- a/crates/ruff_python_parser/src/semantic_errors.rs +++ b/crates/ruff_python_parser/src/semantic_errors.rs @@ -3,12 +3,13 @@ //! This checker is not responsible for traversing the AST itself. Instead, its //! [`SemanticSyntaxChecker::visit_stmt`] and [`SemanticSyntaxChecker::visit_expr`] methods should //! be called in a parent `Visitor`'s `visit_stmt` and `visit_expr` methods, respectively. + use ruff_python_ast::{ self as ast, Expr, ExprContext, IrrefutablePatternKind, Pattern, PythonVersion, Stmt, StmtExpr, - StmtImportFrom, + StmtFunctionDef, StmtImportFrom, comparable::ComparableExpr, helpers, - visitor::{Visitor, walk_expr}, + visitor::{Visitor, walk_expr, walk_stmt}, }; use ruff_text_size::{Ranged, TextRange, TextSize}; use rustc_hash::{FxBuildHasher, FxHashSet}; @@ -739,7 +740,21 @@ impl SemanticSyntaxChecker { self.seen_futures_boundary = true; } } - Stmt::FunctionDef(_) => { + Stmt::FunctionDef(StmtFunctionDef { is_async, body, .. }) => { + if *is_async { + let mut visitor = ReturnVisitor::default(); + visitor.visit_body(body); + + if visitor.has_yield { + if let Some(return_range) = visitor.return_range { + Self::add_error( + ctx, + SemanticSyntaxErrorKind::ReturnInGenerator, + return_range, + ); + } + } + } self.seen_futures_boundary = true; } _ => { @@ -1213,6 +1228,9 @@ impl Display for SemanticSyntaxError { SemanticSyntaxErrorKind::NonlocalWithoutBinding(name) => { write!(f, "no binding for nonlocal `{name}` found") } + SemanticSyntaxErrorKind::ReturnInGenerator => { + write!(f, "`return` with value in async generator") + } } } } @@ -1619,6 +1637,9 @@ pub enum SemanticSyntaxErrorKind { /// Represents a default type parameter followed by a non-default type parameter. TypeParameterDefaultOrder(String), + + /// Represents a `return` statement with a value in an asynchronous generator. + ReturnInGenerator, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, get_size2::GetSize)] @@ -1735,6 +1756,40 @@ impl Visitor<'_> for ReboundComprehensionVisitor<'_> { } } +#[derive(Default)] +struct ReturnVisitor { + return_range: Option, + has_yield: bool, +} + +impl Visitor<'_> for ReturnVisitor { + fn visit_stmt(&mut self, stmt: &Stmt) { + match stmt { + // Do not recurse into nested functions; they're evaluated separately. + Stmt::FunctionDef(_) | Stmt::ClassDef(_) => {} + Stmt::Return(ast::StmtReturn { + value: Some(_), + range, + .. + }) => { + self.return_range = Some(*range); + walk_stmt(self, stmt); + } + _ => walk_stmt(self, stmt), + } + } + + fn visit_expr(&mut self, expr: &Expr) { + match expr { + Expr::Lambda(_) => {} + Expr::Yield(_) | Expr::YieldFrom(_) => { + self.has_yield = true; + } + _ => walk_expr(self, expr), + } + } +} + struct MatchPatternVisitor<'a, Ctx> { names: FxHashSet<&'a ast::name::Name>, ctx: &'a Ctx, From 0280949000ee448f17a1c2b5d802d9ad569301ac Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Dec 2025 19:01:42 +0000 Subject: [PATCH 05/41] [ty] fix panic when attempting to infer the variance of a PEP-695 class that depends on a recursive type aliases and also somehow protocols (#21778) Fixes https://github.com/astral-sh/ty/issues/1716. ## Test plan I added a corpus snippet that causes us to panic on `main` (I tested by running `cargo run -p ty_python_semantic --test=corpus` without the fix applied). --- .../resources/corpus/cyclic_pep695_variance.py | 14 ++++++++++++++ crates/ty_python_semantic/src/types/class.rs | 11 ++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 crates/ty_python_semantic/resources/corpus/cyclic_pep695_variance.py diff --git a/crates/ty_python_semantic/resources/corpus/cyclic_pep695_variance.py b/crates/ty_python_semantic/resources/corpus/cyclic_pep695_variance.py new file mode 100644 index 0000000000..70f354337d --- /dev/null +++ b/crates/ty_python_semantic/resources/corpus/cyclic_pep695_variance.py @@ -0,0 +1,14 @@ +from typing import Protocol + +class A(Protocol): + @property + def f(self): ... + +type Recursive = int | tuple[Recursive, ...] + +class B[T: A]: ... + +class C[T: A](A): + x: tuple[Recursive, ...] + +class D(B[C]): ... diff --git a/crates/ty_python_semantic/src/types/class.rs b/crates/ty_python_semantic/src/types/class.rs index 0abb33c54f..5ebf92fb06 100644 --- a/crates/ty_python_semantic/src/types/class.rs +++ b/crates/ty_python_semantic/src/types/class.rs @@ -340,9 +340,18 @@ impl<'db> From> for Type<'db> { } } +fn variance_of_cycle_initial<'db>( + _db: &'db dyn Db, + _id: salsa::Id, + _self: GenericAlias<'db>, + _typevar: BoundTypeVarInstance<'db>, +) -> TypeVarVariance { + TypeVarVariance::Bivariant +} + #[salsa::tracked] impl<'db> VarianceInferable<'db> for GenericAlias<'db> { - #[salsa::tracked(heap_size=ruff_memory_usage::heap_size)] + #[salsa::tracked(heap_size=ruff_memory_usage::heap_size, cycle_initial=variance_of_cycle_initial)] fn variance_of(self, db: &'db dyn Db, typevar: BoundTypeVarInstance<'db>) -> TypeVarVariance { let origin = self.origin(db); From 45ac30a4d762c9c564d45bb66df813666682e78e Mon Sep 17 00:00:00 2001 From: Aria Desires Date: Wed, 3 Dec 2025 15:04:36 -0500 Subject: [PATCH 06/41] [ty] Teach `ty` the meaning of desperation (try ancestor `pyproject.toml`s as search-paths if module resolution fails) (#21745) ## Summary This makes an importing file a required argument to module resolution, and if the fast-path cached query fails to resolve the module, take the slow-path uncached (could be cached if we want) `desperately_resolve_module` which will walk up from the importing file until it finds a `pyproject.toml` (arbitrary decision, we could try every ancestor directory), at which point it takes one last desperate attempt to use that directory as a search-path. We do not continue walking up once we've found a `pyproject.toml` (arbitrary decision, we could keep going up). Running locally, this fixes every broken-for-workspace-reasons import in pyx's workspace! * Fixes https://github.com/astral-sh/ty/issues/1539 * Improves https://github.com/astral-sh/ty/issues/839 ## Test Plan The workspace tests see a huge improvement on most absolute imports. --- crates/ruff_graph/src/lib.rs | 2 +- crates/ruff_graph/src/resolver.rs | 27 +- crates/ty/tests/file_watching.rs | 70 ++-- crates/ty_ide/src/all_symbols.rs | 2 +- .../resources/mdtest/import/workspaces.md | 364 +++++++++--------- crates/ty_python_semantic/src/dunder_all.rs | 2 +- crates/ty_python_semantic/src/lib.rs | 4 +- .../src/module_resolver/mod.rs | 5 +- .../src/module_resolver/path.rs | 23 +- .../src/module_resolver/resolver.rs | 360 +++++++++++++---- crates/ty_python_semantic/src/place.rs | 10 +- .../src/semantic_index/builder.rs | 2 +- .../src/semantic_index/re_exports.rs | 4 +- .../ty_python_semantic/src/semantic_model.rs | 8 +- crates/ty_python_semantic/src/types.rs | 2 +- crates/ty_python_semantic/src/types/class.rs | 5 +- .../ty_python_semantic/src/types/function.rs | 2 +- .../src/types/ide_support.rs | 11 +- .../src/types/infer/builder.rs | 12 +- .../src/types/special_form.rs | 5 +- crates/ty_test/src/lib.rs | 22 +- 21 files changed, 614 insertions(+), 328 deletions(-) diff --git a/crates/ruff_graph/src/lib.rs b/crates/ruff_graph/src/lib.rs index 64647c8b17..0ada26454f 100644 --- a/crates/ruff_graph/src/lib.rs +++ b/crates/ruff_graph/src/lib.rs @@ -49,7 +49,7 @@ impl ModuleImports { // Resolve the imports. let mut resolved_imports = ModuleImports::default(); for import in imports { - for resolved in Resolver::new(db).resolve(import) { + for resolved in Resolver::new(db, path).resolve(import) { if let Some(path) = resolved.as_system_path() { resolved_imports.insert(path.to_path_buf()); } diff --git a/crates/ruff_graph/src/resolver.rs b/crates/ruff_graph/src/resolver.rs index f1f1589958..b942f81a9a 100644 --- a/crates/ruff_graph/src/resolver.rs +++ b/crates/ruff_graph/src/resolver.rs @@ -1,5 +1,9 @@ -use ruff_db::files::FilePath; -use ty_python_semantic::{ModuleName, resolve_module, resolve_real_module}; +use ruff_db::files::{File, FilePath, system_path_to_file}; +use ruff_db::system::SystemPath; +use ty_python_semantic::{ + ModuleName, resolve_module, resolve_module_confident, resolve_real_module, + resolve_real_module_confident, +}; use crate::ModuleDb; use crate::collector::CollectedImport; @@ -7,12 +11,15 @@ use crate::collector::CollectedImport; /// Collect all imports for a given Python file. pub(crate) struct Resolver<'a> { db: &'a ModuleDb, + file: Option, } impl<'a> Resolver<'a> { /// Initialize a [`Resolver`] with a given [`ModuleDb`]. - pub(crate) fn new(db: &'a ModuleDb) -> Self { - Self { db } + pub(crate) fn new(db: &'a ModuleDb, path: &SystemPath) -> Self { + // If we know the importing file we can potentially resolve more imports + let file = system_path_to_file(db, path).ok(); + Self { db, file } } /// Resolve the [`CollectedImport`] into a [`FilePath`]. @@ -70,13 +77,21 @@ impl<'a> Resolver<'a> { /// Resolves a module name to a module. pub(crate) fn resolve_module(&self, module_name: &ModuleName) -> Option<&'a FilePath> { - let module = resolve_module(self.db, module_name)?; + let module = if let Some(file) = self.file { + resolve_module(self.db, file, module_name)? + } else { + resolve_module_confident(self.db, module_name)? + }; Some(module.file(self.db)?.path(self.db)) } /// Resolves a module name to a module (stubs not allowed). fn resolve_real_module(&self, module_name: &ModuleName) -> Option<&'a FilePath> { - let module = resolve_real_module(self.db, module_name)?; + let module = if let Some(file) = self.file { + resolve_real_module(self.db, file, module_name)? + } else { + resolve_real_module_confident(self.db, module_name)? + }; Some(module.file(self.db)?.path(self.db)) } } diff --git a/crates/ty/tests/file_watching.rs b/crates/ty/tests/file_watching.rs index 5a738b5fd5..5bb80ca857 100644 --- a/crates/ty/tests/file_watching.rs +++ b/crates/ty/tests/file_watching.rs @@ -15,7 +15,7 @@ use ty_project::metadata::pyproject::{PyProject, Tool}; use ty_project::metadata::value::{RangedValue, RelativePathBuf}; use ty_project::watch::{ChangeEvent, ProjectWatcher, directory_watcher}; use ty_project::{Db, ProjectDatabase, ProjectMetadata}; -use ty_python_semantic::{Module, ModuleName, PythonPlatform, resolve_module}; +use ty_python_semantic::{Module, ModuleName, PythonPlatform, resolve_module_confident}; struct TestCase { db: ProjectDatabase, @@ -232,7 +232,8 @@ impl TestCase { } fn module<'c>(&'c self, name: &str) -> Module<'c> { - resolve_module(self.db(), &ModuleName::new(name).unwrap()).expect("module to be present") + resolve_module_confident(self.db(), &ModuleName::new(name).unwrap()) + .expect("module to be present") } fn sorted_submodule_names(&self, parent_module_name: &str) -> Vec { @@ -811,7 +812,8 @@ fn directory_moved_to_project() -> anyhow::Result<()> { .with_context(|| "Failed to create __init__.py")?; std::fs::write(a_original_path.as_std_path(), "").with_context(|| "Failed to create a.py")?; - let sub_a_module = resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()); + let sub_a_module = + resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()); assert_eq!(sub_a_module, None); case.assert_indexed_project_files([bar]); @@ -832,7 +834,9 @@ fn directory_moved_to_project() -> anyhow::Result<()> { .expect("a.py to exist"); // `import sub.a` should now resolve - assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some()); + assert!( + resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some() + ); case.assert_indexed_project_files([bar, init_file, a_file]); @@ -848,7 +852,9 @@ fn directory_moved_to_trash() -> anyhow::Result<()> { ])?; let bar = case.system_file(case.project_path("bar.py")).unwrap(); - assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some()); + assert!( + resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some() + ); let sub_path = case.project_path("sub"); let init_file = case @@ -870,7 +876,9 @@ fn directory_moved_to_trash() -> anyhow::Result<()> { case.apply_changes(changes, None); // `import sub.a` should no longer resolve - assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_none()); + assert!( + resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_none() + ); assert!(!init_file.exists(case.db())); assert!(!a_file.exists(case.db())); @@ -890,8 +898,12 @@ fn directory_renamed() -> anyhow::Result<()> { let bar = case.system_file(case.project_path("bar.py")).unwrap(); - assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some()); - assert!(resolve_module(case.db(), &ModuleName::new_static("foo.baz").unwrap()).is_none()); + assert!( + resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some() + ); + assert!( + resolve_module_confident(case.db(), &ModuleName::new_static("foo.baz").unwrap()).is_none() + ); let sub_path = case.project_path("sub"); let sub_init = case @@ -915,9 +927,13 @@ fn directory_renamed() -> anyhow::Result<()> { case.apply_changes(changes, None); // `import sub.a` should no longer resolve - assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_none()); + assert!( + resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_none() + ); // `import foo.baz` should now resolve - assert!(resolve_module(case.db(), &ModuleName::new_static("foo.baz").unwrap()).is_some()); + assert!( + resolve_module_confident(case.db(), &ModuleName::new_static("foo.baz").unwrap()).is_some() + ); // The old paths are no longer tracked assert!(!sub_init.exists(case.db())); @@ -950,7 +966,9 @@ fn directory_deleted() -> anyhow::Result<()> { let bar = case.system_file(case.project_path("bar.py")).unwrap(); - assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some()); + assert!( + resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some() + ); let sub_path = case.project_path("sub"); @@ -970,7 +988,9 @@ fn directory_deleted() -> anyhow::Result<()> { case.apply_changes(changes, None); // `import sub.a` should no longer resolve - assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_none()); + assert!( + resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_none() + ); assert!(!init_file.exists(case.db())); assert!(!a_file.exists(case.db())); @@ -999,7 +1019,7 @@ fn search_path() -> anyhow::Result<()> { let site_packages = case.root_path().join("site_packages"); assert_eq!( - resolve_module(case.db(), &ModuleName::new("a").unwrap()), + resolve_module_confident(case.db(), &ModuleName::new("a").unwrap()), None ); @@ -1009,7 +1029,7 @@ fn search_path() -> anyhow::Result<()> { case.apply_changes(changes, None); - assert!(resolve_module(case.db(), &ModuleName::new_static("a").unwrap()).is_some()); + assert!(resolve_module_confident(case.db(), &ModuleName::new_static("a").unwrap()).is_some()); case.assert_indexed_project_files([case.system_file(case.project_path("bar.py")).unwrap()]); Ok(()) @@ -1022,7 +1042,7 @@ fn add_search_path() -> anyhow::Result<()> { let site_packages = case.project_path("site_packages"); std::fs::create_dir_all(site_packages.as_std_path())?; - assert!(resolve_module(case.db(), &ModuleName::new_static("a").unwrap()).is_none()); + assert!(resolve_module_confident(case.db(), &ModuleName::new_static("a").unwrap()).is_none()); // Register site-packages as a search path. case.update_options(Options { @@ -1040,7 +1060,7 @@ fn add_search_path() -> anyhow::Result<()> { case.apply_changes(changes, None); - assert!(resolve_module(case.db(), &ModuleName::new_static("a").unwrap()).is_some()); + assert!(resolve_module_confident(case.db(), &ModuleName::new_static("a").unwrap()).is_some()); Ok(()) } @@ -1172,7 +1192,7 @@ fn changed_versions_file() -> anyhow::Result<()> { // Unset the custom typeshed directory. assert_eq!( - resolve_module(case.db(), &ModuleName::new("os").unwrap()), + resolve_module_confident(case.db(), &ModuleName::new("os").unwrap()), None ); @@ -1187,7 +1207,7 @@ fn changed_versions_file() -> anyhow::Result<()> { case.apply_changes(changes, None); - assert!(resolve_module(case.db(), &ModuleName::new("os").unwrap()).is_some()); + assert!(resolve_module_confident(case.db(), &ModuleName::new("os").unwrap()).is_some()); Ok(()) } @@ -1410,7 +1430,7 @@ mod unix { Ok(()) })?; - let baz = resolve_module(case.db(), &ModuleName::new_static("bar.baz").unwrap()) + let baz = resolve_module_confident(case.db(), &ModuleName::new_static("bar.baz").unwrap()) .expect("Expected bar.baz to exist in site-packages."); let baz_project = case.project_path("bar/baz.py"); let baz_file = baz.file(case.db()).unwrap(); @@ -1486,7 +1506,7 @@ mod unix { Ok(()) })?; - let baz = resolve_module(case.db(), &ModuleName::new_static("bar.baz").unwrap()) + let baz = resolve_module_confident(case.db(), &ModuleName::new_static("bar.baz").unwrap()) .expect("Expected bar.baz to exist in site-packages."); let baz_file = baz.file(case.db()).unwrap(); let bar_baz = case.project_path("bar/baz.py"); @@ -1591,7 +1611,7 @@ mod unix { Ok(()) })?; - let baz = resolve_module(case.db(), &ModuleName::new_static("bar.baz").unwrap()) + let baz = resolve_module_confident(case.db(), &ModuleName::new_static("bar.baz").unwrap()) .expect("Expected bar.baz to exist in site-packages."); let baz_site_packages_path = case.project_path(".venv/lib/python3.12/site-packages/bar/baz.py"); @@ -1854,11 +1874,11 @@ fn rename_files_casing_only() -> anyhow::Result<()> { let mut case = setup([("lib.py", "class Foo: ...")])?; assert!( - resolve_module(case.db(), &ModuleName::new("lib").unwrap()).is_some(), + resolve_module_confident(case.db(), &ModuleName::new("lib").unwrap()).is_some(), "Expected `lib` module to exist." ); assert_eq!( - resolve_module(case.db(), &ModuleName::new("Lib").unwrap()), + resolve_module_confident(case.db(), &ModuleName::new("Lib").unwrap()), None, "Expected `Lib` module not to exist" ); @@ -1891,13 +1911,13 @@ fn rename_files_casing_only() -> anyhow::Result<()> { // Resolving `lib` should now fail but `Lib` should now succeed assert_eq!( - resolve_module(case.db(), &ModuleName::new("lib").unwrap()), + resolve_module_confident(case.db(), &ModuleName::new("lib").unwrap()), None, "Expected `lib` module to no longer exist." ); assert!( - resolve_module(case.db(), &ModuleName::new("Lib").unwrap()).is_some(), + resolve_module_confident(case.db(), &ModuleName::new("Lib").unwrap()).is_some(), "Expected `Lib` module to exist" ); diff --git a/crates/ty_ide/src/all_symbols.rs b/crates/ty_ide/src/all_symbols.rs index 5f5774cd69..c7282a5fc7 100644 --- a/crates/ty_ide/src/all_symbols.rs +++ b/crates/ty_ide/src/all_symbols.rs @@ -20,7 +20,7 @@ pub fn all_symbols<'db>( let typing_extensions = ModuleName::new("typing_extensions").unwrap(); let is_typing_extensions_available = importing_from.is_stub(db) - || resolve_real_shadowable_module(db, &typing_extensions).is_some(); + || resolve_real_shadowable_module(db, importing_from, &typing_extensions).is_some(); let results = std::sync::Mutex::new(Vec::new()); { diff --git a/crates/ty_python_semantic/resources/mdtest/import/workspaces.md b/crates/ty_python_semantic/resources/mdtest/import/workspaces.md index 430860a562..4b6eb75165 100644 --- a/crates/ty_python_semantic/resources/mdtest/import/workspaces.md +++ b/crates/ty_python_semantic/resources/mdtest/import/workspaces.md @@ -6,6 +6,15 @@ python file in some random workspace, and so we need to be more tolerant of situ fly in a published package, cases where we're not configured as well as we'd like, or cases where two projects in a monorepo have conflicting definitions (but we want to analyze both at once). +In practice these tests cover what we call "desperate module resolution" which, when an import +fails, results in us walking up the ancestor directories of the importing file and trying those as +"desperate search-paths". + +Currently desperate search-paths are restricted to subdirectories of the first-party search-path +(the directory you're running `ty` in). Currently we only consider one desperate search-path: the +closest ancestor directory containing a `pyproject.toml`. In the future we may want to try every +ancestor `pyproject.toml` or every ancestor directory. + ## Invalid Names While you can't syntactically refer to a module with an invalid name (i.e. one with a `-`, or that @@ -18,9 +27,10 @@ strings and does in fact allow syntactically invalid module names. ### Current File Is Invalid Module Name -Relative and absolute imports should resolve fine in a file that isn't a valid module name. +Relative and absolute imports should resolve fine in a file that isn't a valid module name (in this +case, it could be imported via `importlib.import_module`). -`my-main.py`: +`tests/my-mod.py`: ```py # TODO: there should be no errors in this file @@ -37,13 +47,13 @@ reveal_type(mod2.y) # revealed: Unknown reveal_type(mod3.z) # revealed: int ``` -`mod1.py`: +`tests/mod1.py`: ```py x: int = 1 ``` -`mod2.py`: +`tests/mod2.py`: ```py y: int = 2 @@ -57,13 +67,16 @@ z: int = 2 ### Current Directory Is Invalid Module Name -Relative and absolute imports should resolve fine in a dir that isn't a valid module name. +If python files are rooted in a directory with an invalid module name and they relatively import +each other, there's probably no coherent explanation for what's going on and it's fine that the +relative import don't resolve (but maybe we could provide some good diagnostics). -`my-tests/main.py`: +This is a case that sufficient desperation might "accidentally" make work, so it's included here as +a canary in the coal mine. + +`my-tests/mymod.py`: ```py -# TODO: there should be no errors in this file - # error: [unresolved-import] from .mod1 import x @@ -94,46 +107,97 @@ y: int = 2 z: int = 2 ``` -### Current Directory Is Invalid Package Name +### Ancestor Directory Is Invalid Module Name -Relative and absolute imports should resolve fine in a dir that isn't a valid package name, even if -it contains an `__init__.py`: +Relative and absolute imports *could* resolve fine in the first-party search-path, even if one of +the ancestor dirs is an invalid module. i.e. in this case we will be inclined to compute module +names like `my-proj.tests.mymod`, but it could be that in practice the user always runs this code +rooted in the `my-proj` directory. -`my-tests/__init__.py`: +This case is hard for us to detect and handle in a principled way, but two more extreme kinds of +desperation could handle this: + +- try every ancestor as a desperate search-path +- try the closest ancestor with an invalid module name as a desperate search-path + +The second one is a bit messed up because it could result in situations where someone can get a +worse experience because a directory happened to *not* be invalid as a module name (`myproj` or +`my_proj`). + +`my-proj/tests/mymod.py`: ```py -``` - -`my-tests/main.py`: - -```py -# TODO: there should be no errors in this file +# TODO: it would be *nice* if there were no errors in this file # error: [unresolved-import] from .mod1 import x # error: [unresolved-import] from . import mod2 + +# error: [unresolved-import] import mod3 reveal_type(x) # revealed: Unknown reveal_type(mod2.y) # revealed: Unknown -reveal_type(mod3.z) # revealed: int +reveal_type(mod3.z) # revealed: Unknown ``` -`my-tests/mod1.py`: +`my-proj/tests/mod1.py`: ```py x: int = 1 ``` -`my-tests/mod2.py`: +`my-proj/tests/mod2.py`: ```py y: int = 2 ``` -`mod3.py`: +`my-proj/mod3.py`: + +```py +z: int = 2 +``` + +### Ancestor Directory Above `pyproject.toml` is invalid + +Like the previous tests but with a `pyproject.toml` existing between the invalid name and the python +files. This is an "easier" case in case we use the `pyproject.toml` as a hint about what's going on. + +`my-proj/pyproject.toml`: + +```text +name = "my_proj" +version = "0.1.0" +``` + +`my-proj/tests/main.py`: + +```py +from .mod1 import x +from . import mod2 +import mod3 + +reveal_type(x) # revealed: int +reveal_type(mod2.y) # revealed: int +reveal_type(mod3.z) # revealed: int +``` + +`my-proj/tests/mod1.py`: + +```py +x: int = 1 +``` + +`my-proj/tests/mod2.py`: + +```py +y: int = 2 +``` + +`my-proj/mod3.py`: ```py z: int = 2 @@ -141,7 +205,7 @@ z: int = 2 ## Multiple Projects -It's common for a monorepo to define many separate projects that may or may not depend on eachother +It's common for a monorepo to define many separate projects that may or may not depend on each other and are stitched together with a package manager like `uv` or `poetry`, often as editables. In this case, especially when running as an LSP, we want to be able to analyze all of the projects at once, allowing us to reuse results between projects, without getting confused about things that only make @@ -150,7 +214,7 @@ sense when analyzing the project separately. The following tests will feature two projects, `a` and `b` where the "real" packages are found under `src/` subdirectories (and we've been configured to understand that), but each project also contains other python files in their roots or subdirectories that contains python files which relatively -import eachother and also absolutely import the main package of the project. All of these imports +import each other and also absolutely import the main package of the project. All of these imports *should* resolve. Often the fact that there is both an `a` and `b` project seemingly won't matter, but many possible @@ -164,13 +228,36 @@ following examples include them in case they help. Here we have fairly typical situation where there are two projects `aproj` and `bproj` where the "real" packages are found under `src/` subdirectories, but each project also contains a `tests/` -directory that contains python files which relatively import eachother and also absolutely import +directory that contains python files which relatively import each other and also absolutely import the package they test. All of these imports *should* resolve. ```toml [environment] -# This is similar to what we would compute for installed editables -extra-paths = ["aproj/src/", "bproj/src/"] +# Setup a venv with editables for aproj/src/ and bproj/src/ +python = "/.venv" +``` + +`/.venv/pyvenv.cfg`: + +```cfg +home = /doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin +``` + +`/doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin/python`: + +```text +``` + +`/.venv//a.pth`: + +```pth +aproj/src/ +``` + +`/.venv//b.pth`: + +```pth +bproj/src/ ``` `aproj/tests/test1.py`: @@ -239,16 +326,60 @@ version = "0.1.0" y: str = "20" ``` -### Tests Directory With Ambiguous Project Directories +### Tests Directory With Ambiguous Project Directories Via Editables The same situation as the previous test but instead of the project `a` being in a directory `aproj` to disambiguate, we now need to avoid getting confused about whether `a/` or `a/src/a/` is the package `a` while still resolving imports. +Unfortunately this is a quite difficult square to circle as `a/` is a namespace package of `a` and +`a/src/a/` is a regular package of `a`. **This is a very bad situation you're not supposed to ever +create, and we are now very sensitive to precise search-path ordering.** + +Here the use of editables means that `a/` has higher priority than `a/src/a/`. + +Somehow this results in `a/tests/test1.py` being able to resolve `.setup` but not `.`. + +My best guess is that in this state we can resolve regular modules in `a/tests/` but not namespace +packages because we have some extra validation for namespace packages conflicted by regular +packages, but that validation isn't applied when we successfully resolve a submodule of the +namespace package. + +In this case, as we find that `a/tests/test1.py` matches on the first-party path as `a.tests.test1` +and is syntactically valid. We then resolve `a.tests.test1` and because the namespace package +(`/a/`) comes first we succeed. We then syntactically compute `.` to be `a.tests`. + +When we go to lookup `a.tests.setup`, whatever grace that allowed `a.tests.test1` to resolve still +works so it resolves too. However when we try to resolve `a.tests` on its own some additional +validation rejects the namespace package conflicting with the regular package. + ```toml [environment] -# This is similar to what we would compute for installed editables -extra-paths = ["a/src/", "b/src/"] +# Setup a venv with editables for a/src/ and b/src/ +python = "/.venv" +``` + +`/.venv/pyvenv.cfg`: + +```cfg +home = /doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin +``` + +`/doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin/python`: + +```text +``` + +`/.venv//a.pth`: + +```pth +a/src/ +``` + +`/.venv//b.pth`: + +```pth +b/src/ ``` `a/tests/test1.py`: @@ -256,7 +387,6 @@ extra-paths = ["a/src/", "b/src/"] ```py # TODO: there should be no errors in this file. -# error: [unresolved-import] from .setup import x # error: [unresolved-import] @@ -264,7 +394,7 @@ from . import setup from a import y import a -reveal_type(x) # revealed: Unknown +reveal_type(x) # revealed: int reveal_type(setup.x) # revealed: Unknown reveal_type(y) # revealed: int reveal_type(a.y) # revealed: int @@ -294,7 +424,6 @@ y: int = 10 ```py # TODO: there should be no errors in this file -# error: [unresolved-import] from .setup import x # error: [unresolved-import] @@ -302,7 +431,7 @@ from . import setup from b import y import b -reveal_type(x) # revealed: Unknown +reveal_type(x) # revealed: str reveal_type(setup.x) # revealed: Unknown reveal_type(y) # revealed: str reveal_type(b.y) # revealed: str @@ -327,10 +456,15 @@ version = "0.1.0" y: str = "20" ``` -### Tests Package With Ambiguous Project Directories +### Tests Directory With Ambiguous Project Directories Via `extra-paths` -The same situation as the previous test but `tests/__init__.py` is also defined, in case that -complicates the situation. +The same situation as the previous test but instead of using editables we use `extra-paths` which +have higher priority than the first-party search-path. Thus, `/a/src/a/` is always seen before +`/a/`. + +In this case everything works well because the namespace package `a.tests` (`a/tests/`) is +completely hidden by the regular package `a` (`a/src/a/`) and so we immediately enter desperate +resolution and use the now-unambiguous namespace package `tests`. ```toml [environment] @@ -340,27 +474,17 @@ extra-paths = ["a/src/", "b/src/"] `a/tests/test1.py`: ```py -# TODO: there should be no errors in this file. - -# error: [unresolved-import] from .setup import x - -# error: [unresolved-import] from . import setup from a import y import a -reveal_type(x) # revealed: Unknown -reveal_type(setup.x) # revealed: Unknown +reveal_type(x) # revealed: int +reveal_type(setup.x) # revealed: int reveal_type(y) # revealed: int reveal_type(a.y) # revealed: int ``` -`a/tests/__init__.py`: - -```py -``` - `a/tests/setup.py`: ```py @@ -383,27 +507,17 @@ y: int = 10 `b/tests/test1.py`: ```py -# TODO: there should be no errors in this file - -# error: [unresolved-import] from .setup import x - -# error: [unresolved-import] from . import setup from b import y import b -reveal_type(x) # revealed: Unknown -reveal_type(setup.x) # revealed: Unknown +reveal_type(x) # revealed: str +reveal_type(setup.x) # revealed: str reveal_type(y) # revealed: str reveal_type(b.y) # revealed: str ``` -`b/tests/__init__.py`: - -```py -``` - `b/tests/setup.py`: ```py @@ -431,21 +545,16 @@ that `import main` and expect that to work. `a/tests/test1.py`: ```py -# TODO: there should be no errors in this file. - from .setup import x from . import setup -# error: [unresolved-import] from main import y - -# error: [unresolved-import] import main reveal_type(x) # revealed: int reveal_type(setup.x) # revealed: int -reveal_type(y) # revealed: Unknown -reveal_type(main.y) # revealed: Unknown +reveal_type(y) # revealed: int +reveal_type(main.y) # revealed: int ``` `a/tests/setup.py`: @@ -470,113 +579,16 @@ y: int = 10 `b/tests/test1.py`: ```py -# TODO: there should be no errors in this file - from .setup import x from . import setup -# error: [unresolved-import] from main import y - -# error: [unresolved-import] import main reveal_type(x) # revealed: str reveal_type(setup.x) # revealed: str -reveal_type(y) # revealed: Unknown -reveal_type(main.y) # revealed: Unknown -``` - -`b/tests/setup.py`: - -```py -x: str = "2" -``` - -`b/pyproject.toml`: - -```text -name = "a" -version = "0.1.0" -``` - -`b/main.py`: - -```py -y: str = "20" -``` - -### Tests Package Absolute Importing `main.py` - -The same as the previous case but `tests/__init__.py` exists in case that causes different issues. - -`a/tests/test1.py`: - -```py -# TODO: there should be no errors in this file. - -from .setup import x -from . import setup - -# error: [unresolved-import] -from main import y - -# error: [unresolved-import] -import main - -reveal_type(x) # revealed: int -reveal_type(setup.x) # revealed: int -reveal_type(y) # revealed: Unknown -reveal_type(main.y) # revealed: Unknown -``` - -`a/tests/__init__.py`: - -```py -``` - -`a/tests/setup.py`: - -```py -x: int = 1 -``` - -`a/pyproject.toml`: - -```text -name = "a" -version = "0.1.0" -``` - -`a/main.py`: - -```py -y: int = 10 -``` - -`b/tests/test1.py`: - -```py -# TODO: there should be no errors in this file - -from .setup import x -from . import setup - -# error: [unresolved-import] -from main import y - -# error: [unresolved-import] -import main - -reveal_type(x) # revealed: str -reveal_type(setup.x) # revealed: str -reveal_type(y) # revealed: Unknown -reveal_type(main.y) # revealed: Unknown -``` - -`b/tests/__init__.py`: - -```py +reveal_type(y) # revealed: str +reveal_type(main.y) # revealed: str ``` `b/tests/setup.py`: @@ -606,16 +618,11 @@ imports it. `a/main.py`: ```py -# TODO: there should be no errors in this file. - -# error: [unresolved-import] from utils import x - -# error: [unresolved-import] import utils -reveal_type(x) # revealed: Unknown -reveal_type(utils.x) # revealed: Unknown +reveal_type(x) # revealed: int +reveal_type(utils.x) # revealed: int ``` `a/utils/__init__.py`: @@ -634,16 +641,11 @@ version = "0.1.0" `b/main.py`: ```py -# TODO: there should be no errors in this file. - -# error: [unresolved-import] from utils import x - -# error: [unresolved-import] import utils -reveal_type(x) # revealed: Unknown -reveal_type(utils.x) # revealed: Unknown +reveal_type(x) # revealed: str +reveal_type(utils.x) # revealed: str ``` `b/utils/__init__.py`: diff --git a/crates/ty_python_semantic/src/dunder_all.rs b/crates/ty_python_semantic/src/dunder_all.rs index 17f1706b7e..1803037d6d 100644 --- a/crates/ty_python_semantic/src/dunder_all.rs +++ b/crates/ty_python_semantic/src/dunder_all.rs @@ -166,7 +166,7 @@ impl<'db> DunderAllNamesCollector<'db> { ) -> Option<&'db FxHashSet> { let module_name = ModuleName::from_import_statement(self.db, self.file, import_from).ok()?; - let module = resolve_module(self.db, &module_name)?; + let module = resolve_module(self.db, self.file, &module_name)?; dunder_all_names(self.db, module.file(self.db)?) } diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index 8c1776e154..cf386839c9 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -13,8 +13,8 @@ pub use diagnostic::add_inferred_python_version_hint_to_diagnostic; pub use module_name::{ModuleName, ModuleNameResolutionError}; pub use module_resolver::{ KnownModule, Module, SearchPath, SearchPathValidationError, SearchPaths, all_modules, - list_modules, resolve_module, resolve_real_module, resolve_real_shadowable_module, - system_module_search_paths, + list_modules, resolve_module, resolve_module_confident, resolve_real_module, + resolve_real_module_confident, resolve_real_shadowable_module, system_module_search_paths, }; pub use program::{ Program, ProgramSettings, PythonVersionFileSource, PythonVersionSource, diff --git a/crates/ty_python_semantic/src/module_resolver/mod.rs b/crates/ty_python_semantic/src/module_resolver/mod.rs index cc541d9b31..08f6ee3fc4 100644 --- a/crates/ty_python_semantic/src/module_resolver/mod.rs +++ b/crates/ty_python_semantic/src/module_resolver/mod.rs @@ -6,7 +6,10 @@ pub use module::Module; pub use path::{SearchPath, SearchPathValidationError}; pub use resolver::SearchPaths; pub(crate) use resolver::file_to_module; -pub use resolver::{resolve_module, resolve_real_module, resolve_real_shadowable_module}; +pub use resolver::{ + resolve_module, resolve_module_confident, resolve_real_module, resolve_real_module_confident, + resolve_real_shadowable_module, +}; use ruff_db::system::SystemPath; use crate::Db; diff --git a/crates/ty_python_semantic/src/module_resolver/path.rs b/crates/ty_python_semantic/src/module_resolver/path.rs index cb524cb4ac..5200396dc1 100644 --- a/crates/ty_python_semantic/src/module_resolver/path.rs +++ b/crates/ty_python_semantic/src/module_resolver/path.rs @@ -608,6 +608,18 @@ impl SearchPath { #[must_use] pub(crate) fn relativize_system_path(&self, path: &SystemPath) -> Option { + self.relativize_system_path_only(path) + .map(|relative_path| ModulePath { + search_path: self.clone(), + relative_path: relative_path.as_utf8_path().to_path_buf(), + }) + } + + #[must_use] + pub(crate) fn relativize_system_path_only<'a>( + &self, + path: &'a SystemPath, + ) -> Option<&'a SystemPath> { if path .extension() .is_some_and(|extension| !self.is_valid_extension(extension)) @@ -621,14 +633,7 @@ impl SearchPath { | SearchPathInner::StandardLibraryCustom(search_path) | SearchPathInner::StandardLibraryReal(search_path) | SearchPathInner::SitePackages(search_path) - | SearchPathInner::Editable(search_path) => { - path.strip_prefix(search_path) - .ok() - .map(|relative_path| ModulePath { - search_path: self.clone(), - relative_path: relative_path.as_utf8_path().to_path_buf(), - }) - } + | SearchPathInner::Editable(search_path) => path.strip_prefix(search_path).ok(), SearchPathInner::StandardLibraryVendored(_) => None, } } @@ -783,7 +788,7 @@ impl fmt::Display for SearchPath { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub(super) enum SystemOrVendoredPathRef<'db> { System(&'db SystemPath), Vendored(&'db VendoredPath), diff --git a/crates/ty_python_semantic/src/module_resolver/resolver.rs b/crates/ty_python_semantic/src/module_resolver/resolver.rs index ecf92d2d83..123b4ac31e 100644 --- a/crates/ty_python_semantic/src/module_resolver/resolver.rs +++ b/crates/ty_python_semantic/src/module_resolver/resolver.rs @@ -1,8 +1,31 @@ /*! -This module principally provides two routines for resolving a particular module -name to a `Module`: [`resolve_module`] and [`resolve_real_module`]. You'll -usually want the former, unless you're certain you want to forbid stubs, in -which case, use the latter. +This module principally provides several routines for resolving a particular module +name to a `Module`: + +* [`file_to_module`][]: resolves the module `.` (often as the first step in resolving `.`) +* [`resolve_module`][]: resolves an absolute module name + +You may notice that we actually provide `resolve_(real)_(shadowable)_module_(confident)`. +You almost certainly just want [`resolve_module`][]. The other variations represent +restrictions to answer specific kinds of questions, usually to empower IDE features. + +* The `real` variation disallows all stub files, including the vendored typeshed. + This enables the goto-definition ("real") vs goto-declaration ("stub or real") distinction. + +* The `confident` variation disallows "desperate resolution", which is a fallback + mode where we start trying to use ancestor directories of the importing file + as search-paths, but only if we failed to resolve it with the normal search-paths. + This is mostly just a convenience for cases where we don't want to try to define + the importing file (resolving a `KnownModule` and tests). + +* The `shadowable` variation disables some guards that prevents third-party code + from shadowing any vendored non-stdlib `KnownModule`. In particular `typing_extensions`, + which we vendor and heavily assume the contents of (and so don't ever want to shadow). + This enables checking if the user *actually* has `typing_extensions` installed, + in which case it's ok to suggest it in features like auto-imports. + +There is some awkwardness to the structure of the code to specifically enable caching +of queries, as module resolution happens a lot and involves a lot of disk access. For implementors, see `import-resolution-diagram.svg` for a flow diagram that specifies ty's implementation of Python's import resolution algorithm. @@ -33,14 +56,51 @@ use super::module::{Module, ModuleKind}; use super::path::{ModulePath, SearchPath, SearchPathValidationError, SystemOrVendoredPathRef}; /// Resolves a module name to a module. -pub fn resolve_module<'db>(db: &'db dyn Db, module_name: &ModuleName) -> Option> { +pub fn resolve_module<'db>( + db: &'db dyn Db, + importing_file: File, + module_name: &ModuleName, +) -> Option> { + let interned_name = ModuleNameIngredient::new(db, module_name, ModuleResolveMode::StubsAllowed); + + resolve_module_query(db, interned_name) + .or_else(|| desperately_resolve_module(db, importing_file, interned_name)) +} + +/// Resolves a module name to a module, without desperate resolution available. +/// +/// This is appropriate for resolving a `KnownModule`, or cases where for whatever reason +/// we don't have a well-defined importing file. +pub fn resolve_module_confident<'db>( + db: &'db dyn Db, + module_name: &ModuleName, +) -> Option> { let interned_name = ModuleNameIngredient::new(db, module_name, ModuleResolveMode::StubsAllowed); resolve_module_query(db, interned_name) } /// Resolves a module name to a module (stubs not allowed). -pub fn resolve_real_module<'db>(db: &'db dyn Db, module_name: &ModuleName) -> Option> { +pub fn resolve_real_module<'db>( + db: &'db dyn Db, + importing_file: File, + module_name: &ModuleName, +) -> Option> { + let interned_name = + ModuleNameIngredient::new(db, module_name, ModuleResolveMode::StubsNotAllowed); + + resolve_module_query(db, interned_name) + .or_else(|| desperately_resolve_module(db, importing_file, interned_name)) +} + +/// Resolves a module name to a module, without desperate resolution available (stubs not allowed). +/// +/// This is appropriate for resolving a `KnownModule`, or cases where for whatever reason +/// we don't have a well-defined importing file. +pub fn resolve_real_module_confident<'db>( + db: &'db dyn Db, + module_name: &ModuleName, +) -> Option> { let interned_name = ModuleNameIngredient::new(db, module_name, ModuleResolveMode::StubsNotAllowed); @@ -60,6 +120,7 @@ pub fn resolve_real_module<'db>(db: &'db dyn Db, module_name: &ModuleName) -> Op /// are involved in an import cycle with `builtins`. pub fn resolve_real_shadowable_module<'db>( db: &'db dyn Db, + importing_file: File, module_name: &ModuleName, ) -> Option> { let interned_name = ModuleNameIngredient::new( @@ -69,6 +130,7 @@ pub fn resolve_real_shadowable_module<'db>( ); resolve_module_query(db, interned_name) + .or_else(|| desperately_resolve_module(db, importing_file, interned_name)) } /// Which files should be visible when doing a module query @@ -181,6 +243,55 @@ fn resolve_module_query<'db>( Some(module) } +/// Like `resolve_module_query` but for cases where it failed to resolve the module +/// and we are now Getting Desperate and willing to try the ancestor directories of +/// the `importing_file` as potential temporary search paths that are private +/// to this import. +/// +/// The reason this is split out is because in 99.9% of cases `resolve_module_query` +/// will find the right answer (or no valid answer exists), and we want it to be +/// aggressively cached. Including the `importing_file` as part of that query would +/// trash the caching of import resolution between files. +/// +/// TODO: should (some) of this also be cached? If an entire directory of python files +/// is misunderstood we'll end up in here a lot. +fn desperately_resolve_module<'db>( + db: &'db dyn Db, + importing_file: File, + module_name: ModuleNameIngredient<'db>, +) -> Option> { + let name = module_name.name(db); + let mode = module_name.mode(db); + let _span = tracing::trace_span!("desperately_resolve_module", %name).entered(); + + let Some(resolved) = desperately_resolve_name(db, importing_file, name, mode) else { + tracing::debug!("Module `{name}` not found while looking in parent dirs"); + return None; + }; + + let module = match resolved { + ResolvedName::FileModule(module) => { + tracing::trace!( + "Resolved module `{name}` to `{path}`", + path = module.file.path(db) + ); + Module::file_module( + db, + name.clone(), + module.kind, + module.search_path, + module.file, + ) + } + ResolvedName::NamespacePackage => { + tracing::trace!("Module `{name}` is a namespace package"); + Module::namespace_package(db, name.clone()) + } + }; + + Some(module) +} + /// Resolves the module for the given path. /// /// Returns `None` if the path is not a module locatable via any of the known search paths. @@ -201,13 +312,33 @@ pub(crate) fn path_to_module<'db>(db: &'db dyn Db, path: &FilePath) -> Option` in the file itself, +/// and indeed, one of its primary jobs is resolving `.` to derive the module name of `.`. +/// This intuition is particularly useful for understanding why it's correct that we pass +/// the file itself as `importing_file` to various subroutines. #[salsa::tracked(heap_size=ruff_memory_usage::heap_size)] pub(crate) fn file_to_module(db: &dyn Db, file: File) -> Option> { let _span = tracing::trace_span!("file_to_module", ?file).entered(); let path = SystemOrVendoredPathRef::try_from_file(db, file)?; - let module_name = search_paths(db, ModuleResolveMode::StubsAllowed).find_map(|candidate| { + file_to_module_impl( + db, + file, + path, + search_paths(db, ModuleResolveMode::StubsAllowed), + ) + .or_else(|| file_to_module_impl(db, file, path, desperate_search_paths(db, file).iter())) +} + +fn file_to_module_impl<'db, 'a>( + db: &'db dyn Db, + file: File, + path: SystemOrVendoredPathRef<'a>, + mut search_paths: impl Iterator, +) -> Option> { + let module_name = search_paths.find_map(|candidate: &SearchPath| { let relative_path = match path { SystemOrVendoredPathRef::System(path) => candidate.relativize_system_path(path), SystemOrVendoredPathRef::Vendored(path) => candidate.relativize_vendored_path(path), @@ -219,7 +350,7 @@ pub(crate) fn file_to_module(db: &dyn Db, file: File) -> Option> { // If it doesn't, then that means that multiple modules have the same name in different // root paths, but that the module corresponding to `path` is in a lower priority search path, // in which case we ignore it. - let module = resolve_module(db, &module_name)?; + let module = resolve_module(db, file, &module_name)?; let module_file = module.file(db)?; if file.path(db) == module_file.path(db) { @@ -230,7 +361,7 @@ pub(crate) fn file_to_module(db: &dyn Db, file: File) -> Option> { // If a .py and .pyi are both defined, the .pyi will be the one returned by `resolve_module().file`, // which would make us erroneously believe the `.py` is *not* also this module (breaking things // like relative imports). So here we try `resolve_real_module().file` to cover both cases. - let module = resolve_real_module(db, &module_name)?; + let module = resolve_real_module(db, file, &module_name)?; let module_file = module.file(db)?; if file.path(db) == module_file.path(db) { return Some(module); @@ -250,6 +381,58 @@ pub(crate) fn search_paths(db: &dyn Db, resolve_mode: ModuleResolveMode) -> Sear Program::get(db).search_paths(db).iter(db, resolve_mode) } +/// Get the search-paths that should be used for desperate resolution of imports in this file +/// +/// Currently this is "the closest ancestor dir that contains a pyproject.toml", which is +/// a completely arbitrary decision. We could potentially change this to return an iterator +/// of every ancestor with a pyproject.toml or every ancestor. +/// +/// For now this works well in common cases where we have some larger workspace that contains +/// one or more python projects in sub-directories, and those python projects assume that +/// absolute imports resolve relative to the pyproject.toml they live under. +/// +/// Being so strict minimizes concerns about this going off a lot and doing random +/// chaotic things. In particular, all files under a given pyproject.toml will currently +/// agree on this being their desperate search-path, which is really nice. +#[salsa::tracked(heap_size=ruff_memory_usage::heap_size)] +fn desperate_search_paths(db: &dyn Db, importing_file: File) -> Option { + let system = db.system(); + let importing_path = importing_file.path(db).as_system_path()?; + + // Only allow this if the importing_file is under the first-party search path + let (base_path, rel_path) = + search_paths(db, ModuleResolveMode::StubsAllowed).find_map(|search_path| { + if !search_path.is_first_party() { + return None; + } + Some(( + search_path.as_system_path()?, + search_path.relativize_system_path_only(importing_path)?, + )) + })?; + + // Read the revision on the corresponding file root to + // register an explicit dependency on this directory. When + // the revision gets bumped, the cache that Salsa creates + // for this routine will be invalidated. + // + // (This is conditional because ruff uses this code too and doesn't set roots) + if let Some(root) = db.files().root(db, base_path) { + let _ = root.revision(db); + } + + // Only allow searching up to the first-party path's root + for rel_dir in rel_path.ancestors() { + let candidate_path = base_path.join(rel_dir); + if system.path_exists(&candidate_path.join("pyproject.toml")) + || system.path_exists(&candidate_path.join("ty.toml")) + { + let search_path = SearchPath::first_party(system, candidate_path).ok()?; + return Some(search_path); + } + } + None +} #[derive(Clone, Debug, PartialEq, Eq, get_size2::GetSize)] pub struct SearchPaths { /// Search paths that have been statically determined purely from reading @@ -756,6 +939,30 @@ struct ModuleNameIngredient<'db> { /// Given a module name and a list of search paths in which to lookup modules, /// attempt to resolve the module name fn resolve_name(db: &dyn Db, name: &ModuleName, mode: ModuleResolveMode) -> Option { + let search_paths = search_paths(db, mode); + resolve_name_impl(db, name, mode, search_paths) +} + +/// Like `resolve_name` but for cases where it failed to resolve the module +/// and we are now Getting Desperate and willing to try the ancestor directories of +/// the `importing_file` as potential temporary search paths that are private +/// to this import. +fn desperately_resolve_name( + db: &dyn Db, + importing_file: File, + name: &ModuleName, + mode: ModuleResolveMode, +) -> Option { + let search_paths = desperate_search_paths(db, importing_file); + resolve_name_impl(db, name, mode, search_paths.iter()) +} + +fn resolve_name_impl<'a>( + db: &dyn Db, + name: &ModuleName, + mode: ModuleResolveMode, + search_paths: impl Iterator, +) -> Option { let program = Program::get(db); let python_version = program.python_version(db); let resolver_state = ResolverContext::new(db, python_version, mode); @@ -765,7 +972,7 @@ fn resolve_name(db: &dyn Db, name: &ModuleName, mode: ModuleResolveMode) -> Opti let stub_name = name.to_stub_package(); let mut is_namespace_package = false; - for search_path in search_paths(db, mode) { + for search_path in search_paths { // When a builtin module is imported, standard module resolution is bypassed: // the module name always resolves to the stdlib module, // even if there's a module of the same name in the first-party root @@ -1409,11 +1616,11 @@ mod tests { .build(); let foo_module_name = ModuleName::new_static("foo").unwrap(); - let foo_module = resolve_module(&db, &foo_module_name).unwrap(); + let foo_module = resolve_module_confident(&db, &foo_module_name).unwrap(); assert_eq!( Some(&foo_module), - resolve_module(&db, &foo_module_name).as_ref() + resolve_module_confident(&db, &foo_module_name).as_ref() ); assert_eq!("foo", foo_module.name(&db)); @@ -1435,11 +1642,11 @@ mod tests { .build(); let foo_module_name = ModuleName::new_static("foo").unwrap(); - let foo_module = resolve_module(&db, &foo_module_name).unwrap(); + let foo_module = resolve_module_confident(&db, &foo_module_name).unwrap(); assert_eq!( Some(&foo_module), - resolve_module(&db, &foo_module_name).as_ref() + resolve_module_confident(&db, &foo_module_name).as_ref() ); assert_eq!("foo", foo_module.name(&db)); @@ -1467,11 +1674,11 @@ mod tests { .build(); let foo_module_name = ModuleName::new_static("foo").unwrap(); - let foo_module = resolve_module(&db, &foo_module_name).unwrap(); + let foo_module = resolve_module_confident(&db, &foo_module_name).unwrap(); assert_eq!( Some(&foo_module), - resolve_module(&db, &foo_module_name).as_ref() + resolve_module_confident(&db, &foo_module_name).as_ref() ); assert_eq!("foo", foo_module.name(&db)); @@ -1494,7 +1701,8 @@ mod tests { .build(); let builtins_module_name = ModuleName::new_static("builtins").unwrap(); - let builtins = resolve_module(&db, &builtins_module_name).expect("builtins to resolve"); + let builtins = + resolve_module_confident(&db, &builtins_module_name).expect("builtins to resolve"); assert_eq!( builtins.file(&db).unwrap().path(&db), @@ -1518,7 +1726,8 @@ mod tests { .build(); let builtins_module_name = ModuleName::new_static("builtins").unwrap(); - let builtins = resolve_module(&db, &builtins_module_name).expect("builtins to resolve"); + let builtins = + resolve_module_confident(&db, &builtins_module_name).expect("builtins to resolve"); assert_eq!( builtins.file(&db).unwrap().path(&db), @@ -1539,11 +1748,11 @@ mod tests { .build(); let functools_module_name = ModuleName::new_static("functools").unwrap(); - let functools_module = resolve_module(&db, &functools_module_name).unwrap(); + let functools_module = resolve_module_confident(&db, &functools_module_name).unwrap(); assert_eq!( Some(&functools_module), - resolve_module(&db, &functools_module_name).as_ref() + resolve_module_confident(&db, &functools_module_name).as_ref() ); assert_eq!(&stdlib, functools_module.search_path(&db).unwrap()); @@ -1596,9 +1805,10 @@ mod tests { let existing_modules = create_module_names(&["asyncio", "functools", "xml.etree"]); for module_name in existing_modules { - let resolved_module = resolve_module(&db, &module_name).unwrap_or_else(|| { - panic!("Expected module {module_name} to exist in the mock stdlib") - }); + let resolved_module = + resolve_module_confident(&db, &module_name).unwrap_or_else(|| { + panic!("Expected module {module_name} to exist in the mock stdlib") + }); let search_path = resolved_module.search_path(&db).unwrap(); assert_eq!( &stdlib, search_path, @@ -1649,7 +1859,7 @@ mod tests { for module_name in nonexisting_modules { assert!( - resolve_module(&db, &module_name).is_none(), + resolve_module_confident(&db, &module_name).is_none(), "Unexpectedly resolved a module for {module_name}" ); } @@ -1692,9 +1902,10 @@ mod tests { ]); for module_name in existing_modules { - let resolved_module = resolve_module(&db, &module_name).unwrap_or_else(|| { - panic!("Expected module {module_name} to exist in the mock stdlib") - }); + let resolved_module = + resolve_module_confident(&db, &module_name).unwrap_or_else(|| { + panic!("Expected module {module_name} to exist in the mock stdlib") + }); let search_path = resolved_module.search_path(&db).unwrap(); assert_eq!( &stdlib, search_path, @@ -1728,7 +1939,7 @@ mod tests { let nonexisting_modules = create_module_names(&["importlib", "xml", "xml.etree"]); for module_name in nonexisting_modules { assert!( - resolve_module(&db, &module_name).is_none(), + resolve_module_confident(&db, &module_name).is_none(), "Unexpectedly resolved a module for {module_name}" ); } @@ -1750,11 +1961,11 @@ mod tests { .build(); let functools_module_name = ModuleName::new_static("functools").unwrap(); - let functools_module = resolve_module(&db, &functools_module_name).unwrap(); + let functools_module = resolve_module_confident(&db, &functools_module_name).unwrap(); assert_eq!( Some(&functools_module), - resolve_module(&db, &functools_module_name).as_ref() + resolve_module_confident(&db, &functools_module_name).as_ref() ); assert_eq!(&src, functools_module.search_path(&db).unwrap()); assert_eq!(ModuleKind::Module, functools_module.kind(&db)); @@ -1777,7 +1988,7 @@ mod tests { .build(); let pydoc_data_topics_name = ModuleName::new_static("pydoc_data.topics").unwrap(); - let pydoc_data_topics = resolve_module(&db, &pydoc_data_topics_name).unwrap(); + let pydoc_data_topics = resolve_module_confident(&db, &pydoc_data_topics_name).unwrap(); assert_eq!("pydoc_data.topics", pydoc_data_topics.name(&db)); assert_eq!(pydoc_data_topics.search_path(&db).unwrap(), &stdlib); @@ -1794,7 +2005,8 @@ mod tests { .build(); let foo_path = src.join("foo/__init__.py"); - let foo_module = resolve_module(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); + let foo_module = + resolve_module_confident(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); assert_eq!("foo", foo_module.name(&db)); assert_eq!(&src, foo_module.search_path(&db).unwrap()); @@ -1821,7 +2033,8 @@ mod tests { let TestCase { db, src, .. } = TestCaseBuilder::new().with_src_files(SRC).build(); - let foo_module = resolve_module(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); + let foo_module = + resolve_module_confident(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); let foo_init_path = src.join("foo/__init__.py"); assert_eq!(&src, foo_module.search_path(&db).unwrap()); @@ -1844,8 +2057,9 @@ mod tests { let TestCase { db, src, .. } = TestCaseBuilder::new().with_src_files(SRC).build(); - let foo = resolve_module(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); - let foo_real = resolve_real_module(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); + let foo = resolve_module_confident(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); + let foo_real = + resolve_real_module_confident(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); let foo_stub = src.join("foo.pyi"); assert_eq!(&src, foo.search_path(&db).unwrap()); @@ -1870,7 +2084,7 @@ mod tests { let TestCase { db, src, .. } = TestCaseBuilder::new().with_src_files(SRC).build(); let baz_module = - resolve_module(&db, &ModuleName::new_static("foo.bar.baz").unwrap()).unwrap(); + resolve_module_confident(&db, &ModuleName::new_static("foo.bar.baz").unwrap()).unwrap(); let baz_path = src.join("foo/bar/baz.py"); assert_eq!(&src, baz_module.search_path(&db).unwrap()); @@ -1894,7 +2108,8 @@ mod tests { .with_site_packages_files(&[("foo.py", "")]) .build(); - let foo_module = resolve_module(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); + let foo_module = + resolve_module_confident(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); let foo_src_path = src.join("foo.py"); assert_eq!(&src, foo_module.search_path(&db).unwrap()); @@ -1965,8 +2180,10 @@ mod tests { }, ); - let foo_module = resolve_module(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); - let bar_module = resolve_module(&db, &ModuleName::new_static("bar").unwrap()).unwrap(); + let foo_module = + resolve_module_confident(&db, &ModuleName::new_static("foo").unwrap()).unwrap(); + let bar_module = + resolve_module_confident(&db, &ModuleName::new_static("bar").unwrap()).unwrap(); assert_ne!(foo_module, bar_module); @@ -2001,7 +2218,7 @@ mod tests { .build(); let foo_module_name = ModuleName::new_static("foo").unwrap(); - let foo_module = resolve_module(&db, &foo_module_name).unwrap(); + let foo_module = resolve_module_confident(&db, &foo_module_name).unwrap(); let foo_pieces = ( foo_module.name(&db).clone(), foo_module.file(&db), @@ -2022,7 +2239,7 @@ mod tests { // Re-query the foo module. The foo module should still be cached // because `bar.py` isn't relevant for resolving `foo`. - let foo_module2 = resolve_module(&db, &foo_module_name); + let foo_module2 = resolve_module_confident(&db, &foo_module_name); let foo_pieces2 = foo_module2.map(|foo_module2| { ( foo_module2.name(&db).clone(), @@ -2049,14 +2266,15 @@ mod tests { let foo_path = src.join("foo.py"); let foo_module_name = ModuleName::new_static("foo").unwrap(); - assert_eq!(resolve_module(&db, &foo_module_name), None); + assert_eq!(resolve_module_confident(&db, &foo_module_name), None); // Now write the foo file db.write_file(&foo_path, "x = 1")?; let foo_file = system_path_to_file(&db, &foo_path).expect("foo.py to exist"); - let foo_module = resolve_module(&db, &foo_module_name).expect("Foo module to resolve"); + let foo_module = + resolve_module_confident(&db, &foo_module_name).expect("Foo module to resolve"); assert_eq!(foo_file, foo_module.file(&db).unwrap()); Ok(()) @@ -2070,7 +2288,8 @@ mod tests { let TestCase { mut db, src, .. } = TestCaseBuilder::new().with_src_files(SRC).build(); let foo_module_name = ModuleName::new_static("foo").unwrap(); - let foo_module = resolve_module(&db, &foo_module_name).expect("foo module to exist"); + let foo_module = + resolve_module_confident(&db, &foo_module_name).expect("foo module to exist"); let foo_init_path = src.join("foo/__init__.py"); assert_eq!(&foo_init_path, foo_module.file(&db).unwrap().path(&db)); @@ -2082,7 +2301,8 @@ mod tests { File::sync_path(&mut db, &foo_init_path); File::sync_path(&mut db, foo_init_path.parent().unwrap()); - let foo_module = resolve_module(&db, &foo_module_name).expect("Foo module to resolve"); + let foo_module = + resolve_module_confident(&db, &foo_module_name).expect("Foo module to resolve"); assert_eq!(&src.join("foo.py"), foo_module.file(&db).unwrap().path(&db)); Ok(()) @@ -2108,7 +2328,7 @@ mod tests { let functools_module_name = ModuleName::new_static("functools").unwrap(); let stdlib_functools_path = stdlib.join("functools.pyi"); - let functools_module = resolve_module(&db, &functools_module_name).unwrap(); + let functools_module = resolve_module_confident(&db, &functools_module_name).unwrap(); assert_eq!(functools_module.search_path(&db).unwrap(), &stdlib); assert_eq!( Ok(functools_module.file(&db).unwrap()), @@ -2121,7 +2341,7 @@ mod tests { let site_packages_functools_path = site_packages.join("functools.py"); db.write_file(&site_packages_functools_path, "f: int") .unwrap(); - let functools_module = resolve_module(&db, &functools_module_name).unwrap(); + let functools_module = resolve_module_confident(&db, &functools_module_name).unwrap(); let functools_file = functools_module.file(&db).unwrap(); let functools_search_path = functools_module.search_path(&db).unwrap().clone(); let events = db.take_salsa_events(); @@ -2156,7 +2376,7 @@ mod tests { .build(); let functools_module_name = ModuleName::new_static("functools").unwrap(); - let functools_module = resolve_module(&db, &functools_module_name).unwrap(); + let functools_module = resolve_module_confident(&db, &functools_module_name).unwrap(); assert_eq!(functools_module.search_path(&db).unwrap(), &stdlib); assert_eq!( Ok(functools_module.file(&db).unwrap()), @@ -2167,7 +2387,7 @@ mod tests { // since first-party files take higher priority in module resolution: let src_functools_path = src.join("functools.py"); db.write_file(&src_functools_path, "FOO: int").unwrap(); - let functools_module = resolve_module(&db, &functools_module_name).unwrap(); + let functools_module = resolve_module_confident(&db, &functools_module_name).unwrap(); assert_eq!(functools_module.search_path(&db).unwrap(), &src); assert_eq!( Ok(functools_module.file(&db).unwrap()), @@ -2198,7 +2418,7 @@ mod tests { let functools_module_name = ModuleName::new_static("functools").unwrap(); let src_functools_path = src.join("functools.py"); - let functools_module = resolve_module(&db, &functools_module_name).unwrap(); + let functools_module = resolve_module_confident(&db, &functools_module_name).unwrap(); assert_eq!(functools_module.search_path(&db).unwrap(), &src); assert_eq!( Ok(functools_module.file(&db).unwrap()), @@ -2211,7 +2431,7 @@ mod tests { .remove_file(&src_functools_path) .unwrap(); File::sync_path(&mut db, &src_functools_path); - let functools_module = resolve_module(&db, &functools_module_name).unwrap(); + let functools_module = resolve_module_confident(&db, &functools_module_name).unwrap(); assert_eq!(functools_module.search_path(&db).unwrap(), &stdlib); assert_eq!( Ok(functools_module.file(&db).unwrap()), @@ -2233,8 +2453,8 @@ mod tests { let foo_module_name = ModuleName::new_static("foo").unwrap(); let foo_bar_module_name = ModuleName::new_static("foo.bar").unwrap(); - let foo_module = resolve_module(&db, &foo_module_name).unwrap(); - let foo_bar_module = resolve_module(&db, &foo_bar_module_name).unwrap(); + let foo_module = resolve_module_confident(&db, &foo_module_name).unwrap(); + let foo_bar_module = resolve_module_confident(&db, &foo_bar_module_name).unwrap(); assert_eq!( foo_module.file(&db).unwrap().path(&db), @@ -2262,11 +2482,11 @@ mod tests { // Lines with leading whitespace in `.pth` files do not parse: let foo_module_name = ModuleName::new_static("foo").unwrap(); - assert_eq!(resolve_module(&db, &foo_module_name), None); + assert_eq!(resolve_module_confident(&db, &foo_module_name), None); // Lines with trailing whitespace in `.pth` files do: let bar_module_name = ModuleName::new_static("bar").unwrap(); - let bar_module = resolve_module(&db, &bar_module_name).unwrap(); + let bar_module = resolve_module_confident(&db, &bar_module_name).unwrap(); assert_eq!( bar_module.file(&db).unwrap().path(&db), &FilePath::system("/y/src/bar.py") @@ -2285,7 +2505,7 @@ mod tests { .build(); let foo_module_name = ModuleName::new_static("foo").unwrap(); - let foo_module = resolve_module(&db, &foo_module_name).unwrap(); + let foo_module = resolve_module_confident(&db, &foo_module_name).unwrap(); assert_eq!( foo_module.file(&db).unwrap().path(&db), @@ -2333,10 +2553,10 @@ not_a_directory let b_module_name = ModuleName::new_static("b").unwrap(); let spam_module_name = ModuleName::new_static("spam").unwrap(); - let foo_module = resolve_module(&db, &foo_module_name).unwrap(); - let a_module = resolve_module(&db, &a_module_name).unwrap(); - let b_module = resolve_module(&db, &b_module_name).unwrap(); - let spam_module = resolve_module(&db, &spam_module_name).unwrap(); + let foo_module = resolve_module_confident(&db, &foo_module_name).unwrap(); + let a_module = resolve_module_confident(&db, &a_module_name).unwrap(); + let b_module = resolve_module_confident(&db, &b_module_name).unwrap(); + let spam_module = resolve_module_confident(&db, &spam_module_name).unwrap(); assert_eq!( foo_module.file(&db).unwrap().path(&db), @@ -2370,14 +2590,14 @@ not_a_directory let foo_module_name = ModuleName::new_static("foo").unwrap(); let bar_module_name = ModuleName::new_static("bar").unwrap(); - let foo_module = resolve_module(&db, &foo_module_name).unwrap(); + let foo_module = resolve_module_confident(&db, &foo_module_name).unwrap(); assert_eq!( foo_module.file(&db).unwrap().path(&db), &FilePath::system("/x/src/foo.py") ); db.clear_salsa_events(); - let bar_module = resolve_module(&db, &bar_module_name).unwrap(); + let bar_module = resolve_module_confident(&db, &bar_module_name).unwrap(); assert_eq!( bar_module.file(&db).unwrap().path(&db), &FilePath::system("/y/src/bar.py") @@ -2407,7 +2627,7 @@ not_a_directory db.write_files(x_directory).unwrap(); let foo_module_name = ModuleName::new_static("foo").unwrap(); - let foo_module = resolve_module(&db, &foo_module_name).unwrap(); + let foo_module = resolve_module_confident(&db, &foo_module_name).unwrap(); assert_eq!( foo_module.file(&db).unwrap().path(&db), &FilePath::system("/x/src/foo.py") @@ -2419,7 +2639,7 @@ not_a_directory File::sync_path(&mut db, &site_packages.join("_foo.pth")); - assert_eq!(resolve_module(&db, &foo_module_name), None); + assert_eq!(resolve_module_confident(&db, &foo_module_name), None); } #[test] @@ -2434,7 +2654,7 @@ not_a_directory db.write_files(x_directory).unwrap(); let foo_module_name = ModuleName::new_static("foo").unwrap(); - let foo_module = resolve_module(&db, &foo_module_name).unwrap(); + let foo_module = resolve_module_confident(&db, &foo_module_name).unwrap(); let src_path = SystemPathBuf::from("/x/src"); assert_eq!( foo_module.file(&db).unwrap().path(&db), @@ -2447,7 +2667,7 @@ not_a_directory db.memory_file_system().remove_directory(&src_path).unwrap(); File::sync_path(&mut db, &src_path.join("foo.py")); File::sync_path(&mut db, &src_path); - assert_eq!(resolve_module(&db, &foo_module_name), None); + assert_eq!(resolve_module_confident(&db, &foo_module_name), None); } #[test] @@ -2507,7 +2727,7 @@ not_a_directory // The editable installs discovered from the `.pth` file in the first `site-packages` directory // take precedence over the second `site-packages` directory... let a_module_name = ModuleName::new_static("a").unwrap(); - let a_module = resolve_module(&db, &a_module_name).unwrap(); + let a_module = resolve_module_confident(&db, &a_module_name).unwrap(); assert_eq!( a_module.file(&db).unwrap().path(&db), &editable_install_location @@ -2521,7 +2741,7 @@ not_a_directory // ...But now that the `.pth` file in the first `site-packages` directory has been deleted, // the editable install no longer exists, so the module now resolves to the file in the // second `site-packages` directory - let a_module = resolve_module(&db, &a_module_name).unwrap(); + let a_module = resolve_module_confident(&db, &a_module_name).unwrap(); assert_eq!( a_module.file(&db).unwrap().path(&db), &system_site_packages_location @@ -2579,12 +2799,12 @@ not_a_directory // Now try to resolve the module `A` (note the capital `A` instead of `a`). let a_module_name = ModuleName::new_static("A").unwrap(); - assert_eq!(resolve_module(&db, &a_module_name), None); + assert_eq!(resolve_module_confident(&db, &a_module_name), None); // Now lookup the same module using the lowercase `a` and it should // resolve to the file in the system site-packages let a_module_name = ModuleName::new_static("a").unwrap(); - let a_module = resolve_module(&db, &a_module_name).expect("a.py to resolve"); + let a_module = resolve_module_confident(&db, &a_module_name).expect("a.py to resolve"); assert!( a_module .file(&db) diff --git a/crates/ty_python_semantic/src/place.rs b/crates/ty_python_semantic/src/place.rs index 3cb66efe33..98f7a2b8e4 100644 --- a/crates/ty_python_semantic/src/place.rs +++ b/crates/ty_python_semantic/src/place.rs @@ -1,7 +1,7 @@ use ruff_db::files::File; use crate::dunder_all::dunder_all_names; -use crate::module_resolver::{KnownModule, file_to_module}; +use crate::module_resolver::{KnownModule, file_to_module, resolve_module_confident}; use crate::semantic_index::definition::{Definition, DefinitionState}; use crate::semantic_index::place::{PlaceExprRef, ScopedPlaceId}; use crate::semantic_index::scope::ScopeId; @@ -14,7 +14,7 @@ use crate::types::{ Truthiness, Type, TypeAndQualifiers, TypeQualifiers, UnionBuilder, UnionType, binding_type, declaration_type, todo_type, }; -use crate::{Db, FxOrderSet, Program, resolve_module}; +use crate::{Db, FxOrderSet, Program}; pub(crate) use implicit_globals::{ module_type_implicit_global_declaration, module_type_implicit_global_symbol, @@ -379,7 +379,7 @@ pub(crate) fn imported_symbol<'db>( /// and should not be used when a symbol is being explicitly imported from the `builtins` module /// (e.g. `from builtins import int`). pub(crate) fn builtins_symbol<'db>(db: &'db dyn Db, symbol: &str) -> PlaceAndQualifiers<'db> { - resolve_module(db, &KnownModule::Builtins.name()) + resolve_module_confident(db, &KnownModule::Builtins.name()) .and_then(|module| { let file = module.file(db)?; Some( @@ -409,7 +409,7 @@ pub(crate) fn known_module_symbol<'db>( known_module: KnownModule, symbol: &str, ) -> PlaceAndQualifiers<'db> { - resolve_module(db, &known_module.name()) + resolve_module_confident(db, &known_module.name()) .and_then(|module| { let file = module.file(db)?; Some(imported_symbol(db, file, symbol, None)) @@ -448,7 +448,7 @@ pub(crate) fn builtins_module_scope(db: &dyn Db) -> Option> { /// /// Can return `None` if a custom typeshed is used that is missing the core module in question. fn core_module_scope(db: &dyn Db, core_module: KnownModule) -> Option> { - let module = resolve_module(db, &core_module.name())?; + let module = resolve_module_confident(db, &core_module.name())?; Some(global_scope(db, module.file(db)?)) } diff --git a/crates/ty_python_semantic/src/semantic_index/builder.rs b/crates/ty_python_semantic/src/semantic_index/builder.rs index 9ba2069784..373c80f7ee 100644 --- a/crates/ty_python_semantic/src/semantic_index/builder.rs +++ b/crates/ty_python_semantic/src/semantic_index/builder.rs @@ -1582,7 +1582,7 @@ impl<'ast> Visitor<'ast> for SemanticIndexBuilder<'_, 'ast> { continue; }; - let Some(module) = resolve_module(self.db, &module_name) else { + let Some(module) = resolve_module(self.db, self.file, &module_name) else { continue; }; diff --git a/crates/ty_python_semantic/src/semantic_index/re_exports.rs b/crates/ty_python_semantic/src/semantic_index/re_exports.rs index 22389fc549..693c0f3437 100644 --- a/crates/ty_python_semantic/src/semantic_index/re_exports.rs +++ b/crates/ty_python_semantic/src/semantic_index/re_exports.rs @@ -250,7 +250,9 @@ impl<'db> Visitor<'db> for ExportFinder<'db> { for export in ModuleName::from_import_statement(self.db, self.file, node) .ok() - .and_then(|module_name| resolve_module(self.db, &module_name)) + .and_then(|module_name| { + resolve_module(self.db, self.file, &module_name) + }) .iter() .flat_map(|module| { module diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 2057db47ab..4dc8a59bab 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -100,14 +100,14 @@ impl<'db> SemanticModel<'db> { pub fn resolve_module(&self, module: Option<&str>, level: u32) -> Option> { let module_name = ModuleName::from_identifier_parts(self.db, self.file, module, level).ok()?; - resolve_module(self.db, &module_name) + resolve_module(self.db, self.file, &module_name) } /// Returns completions for symbols available in a `import ` context. pub fn import_completions(&self) -> Vec> { let typing_extensions = ModuleName::new("typing_extensions").unwrap(); let is_typing_extensions_available = self.file.is_stub(self.db) - || resolve_real_shadowable_module(self.db, &typing_extensions).is_some(); + || resolve_real_shadowable_module(self.db, self.file, &typing_extensions).is_some(); list_modules(self.db) .into_iter() .filter(|module| { @@ -146,7 +146,7 @@ impl<'db> SemanticModel<'db> { &self, module_name: &ModuleName, ) -> Vec> { - let Some(module) = resolve_module(self.db, module_name) else { + let Some(module) = resolve_module(self.db, self.file, module_name) else { tracing::debug!("Could not resolve module from `{module_name:?}`"); return vec![]; }; @@ -156,7 +156,7 @@ impl<'db> SemanticModel<'db> { /// Returns completions for symbols available in the given module as if /// it were imported by this model's `File`. fn module_completions(&self, module_name: &ModuleName) -> Vec> { - let Some(module) = resolve_module(self.db, module_name) else { + let Some(module) = resolve_module(self.db, self.file, module_name) else { tracing::debug!("Could not resolve module from `{module_name:?}`"); return vec![]; }; diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index e3b0aa7717..b1cfc9d3c9 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -12684,7 +12684,7 @@ impl<'db> ModuleLiteralType<'db> { let relative_submodule_name = ModuleName::new(name)?; let mut absolute_submodule_name = self.module(db).name(db).clone(); absolute_submodule_name.extend(&relative_submodule_name); - let submodule = resolve_module(db, &absolute_submodule_name)?; + let submodule = resolve_module(db, importing_file, &absolute_submodule_name)?; Some(Type::module_literal(db, importing_file, submodule)) } diff --git a/crates/ty_python_semantic/src/types/class.rs b/crates/ty_python_semantic/src/types/class.rs index 5ebf92fb06..06f91502fc 100644 --- a/crates/ty_python_semantic/src/types/class.rs +++ b/crates/ty_python_semantic/src/types/class.rs @@ -5850,7 +5850,7 @@ impl SlotsKind { mod tests { use super::*; use crate::db::tests::setup_db; - use crate::module_resolver::resolve_module; + use crate::module_resolver::resolve_module_confident; use crate::{PythonVersionSource, PythonVersionWithSource}; use salsa::Setter; use strum::IntoEnumIterator; @@ -5866,7 +5866,8 @@ mod tests { }); for class in KnownClass::iter() { let class_name = class.name(&db); - let class_module = resolve_module(&db, &class.canonical_module(&db).name()).unwrap(); + let class_module = + resolve_module_confident(&db, &class.canonical_module(&db).name()).unwrap(); assert_eq!( KnownClass::try_from_file_and_name( diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index 4437e09698..f8537c27a7 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -1882,7 +1882,7 @@ impl KnownFunction { let Some(module_name) = ModuleName::new(module_name) else { return; }; - let Some(module) = resolve_module(db, &module_name) else { + let Some(module) = resolve_module(db, file, &module_name) else { return; }; diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 2c53ea275f..111edcabc5 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -938,7 +938,7 @@ mod resolve_definition { }; // Resolve the module to its file - let Some(resolved_module) = resolve_module(db, &module_name) else { + let Some(resolved_module) = resolve_module(db, file, &module_name) else { return Vec::new(); // Module not found, return empty list }; @@ -1025,7 +1025,7 @@ mod resolve_definition { else { return Vec::new(); }; - let Some(resolved_module) = resolve_module(db, &module_name) else { + let Some(resolved_module) = resolve_module(db, file, &module_name) else { return Vec::new(); }; resolved_module.file(db) @@ -1134,7 +1134,12 @@ mod resolve_definition { // It's definitely a stub, so now rerun module resolution but with stubs disabled. let stub_module = file_to_module(db, stub_file_for_module_lookup)?; trace!("Found stub module: {}", stub_module.name(db)); - let real_module = resolve_real_module(db, stub_module.name(db))?; + // We need to pass an importing file to `resolve_real_module` which is a bit odd + // here because there isn't really an importing file. However this `resolve_real_module` + // can be understood as essentially `import .`, which is also what `file_to_module` is, + // so this is in fact exactly the file we want to consider the importer. + let real_module = + resolve_real_module(db, stub_file_for_module_lookup, stub_module.name(db))?; trace!("Found real module: {}", real_module.name(db)); let real_file = real_module.file(db)?; trace!("Found real file: {}", real_file.path(db)); diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 4d05719ebf..d45b35298d 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -5935,7 +5935,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ) else { return false; }; - resolve_module(self.db(), &module_name).is_some() + resolve_module(self.db(), self.file(), &module_name).is_some() }) { diagnostic .help("The module can be resolved if the number of leading dots is reduced"); @@ -6172,7 +6172,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } }; - if resolve_module(self.db(), &module_name).is_none() { + if resolve_module(self.db(), self.file(), &module_name).is_none() { self.report_unresolved_import(import_from.into(), module_ref.range(), *level, module); } } @@ -6190,7 +6190,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { return; }; - let Some(module) = resolve_module(self.db(), &module_name) else { + let Some(module) = resolve_module(self.db(), self.file(), &module_name) else { self.add_unknown_declaration_with_binding(alias.into(), definition); return; }; @@ -6375,7 +6375,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.add_binding(import_from.into(), definition, |_, _| Type::unknown()); return; }; - let Some(module) = resolve_module(self.db(), &thispackage_name) else { + let Some(module) = resolve_module(self.db(), self.file(), &thispackage_name) else { self.add_binding(import_from.into(), definition, |_, _| Type::unknown()); return; }; @@ -6606,7 +6606,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } fn module_type_from_name(&self, module_name: &ModuleName) -> Option> { - resolve_module(self.db(), module_name) + resolve_module(self.db(), self.file(), module_name) .map(|module| Type::module_literal(self.db(), self.file(), module)) } @@ -9186,7 +9186,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { { let mut maybe_submodule_name = module_name.clone(); maybe_submodule_name.extend(&relative_submodule); - if resolve_module(db, &maybe_submodule_name).is_some() { + if resolve_module(db, self.file(), &maybe_submodule_name).is_some() { if let Some(builder) = self .context .report_lint(&POSSIBLY_MISSING_ATTRIBUTE, attribute) diff --git a/crates/ty_python_semantic/src/types/special_form.rs b/crates/ty_python_semantic/src/types/special_form.rs index 423e1196bd..4d3fa066ed 100644 --- a/crates/ty_python_semantic/src/types/special_form.rs +++ b/crates/ty_python_semantic/src/types/special_form.rs @@ -3,8 +3,7 @@ use super::{ClassType, Type, class::KnownClass}; use crate::db::Db; -use crate::module_resolver::{KnownModule, file_to_module}; -use crate::resolve_module; +use crate::module_resolver::{KnownModule, file_to_module, resolve_module_confident}; use crate::semantic_index::place::ScopedPlaceId; use crate::semantic_index::{FileScopeId, place_table, use_def_map}; use crate::types::TypeDefinition; @@ -544,7 +543,7 @@ impl SpecialFormType { self.definition_modules() .iter() .find_map(|module| { - let file = resolve_module(db, &module.name())?.file(db)?; + let file = resolve_module_confident(db, &module.name())?.file(db)?; let scope = FileScopeId::global().to_scope_id(db, file); let symbol_id = place_table(db, scope).symbol_id(self.name())?; diff --git a/crates/ty_test/src/lib.rs b/crates/ty_test/src/lib.rs index ad4e7ebe4e..feb38bdf66 100644 --- a/crates/ty_test/src/lib.rs +++ b/crates/ty_test/src/lib.rs @@ -21,7 +21,7 @@ use ty_python_semantic::types::{UNDEFINED_REVEAL, check_types}; use ty_python_semantic::{ Module, Program, ProgramSettings, PythonEnvironment, PythonPlatform, PythonVersionSource, PythonVersionWithSource, SearchPath, SearchPathSettings, SysPrefixPathOrigin, list_modules, - resolve_module, + resolve_module_confident, }; mod assertion; @@ -259,7 +259,10 @@ fn run_test( } assert!( - matches!(embedded.lang, "py" | "pyi" | "python" | "text" | "cfg"), + matches!( + embedded.lang, + "py" | "pyi" | "python" | "text" | "cfg" | "pth" + ), "Supported file types are: py (or python), pyi, text, cfg and ignore" ); @@ -296,7 +299,16 @@ fn run_test( full_path = new_path; } - db.write_file(&full_path, &embedded.code).unwrap(); + let temp_string; + let to_write = if embedded.lang == "pth" && !embedded.code.starts_with('/') { + // Make any relative .pths be relative to src_path + temp_string = format!("{src_path}/{}", embedded.code); + &*temp_string + } else { + &*embedded.code + }; + + db.write_file(&full_path, to_write).unwrap(); if !(full_path.starts_with(&src_path) && matches!(embedded.lang, "py" | "python" | "pyi")) @@ -566,7 +578,9 @@ struct ModuleInconsistency<'db> { fn run_module_resolution_consistency_test(db: &db::Db) -> Result<(), Vec>> { let mut errs = vec![]; for from_list in list_modules(db) { - errs.push(match resolve_module(db, from_list.name(db)) { + // TODO: For now list_modules does not partake in desperate module resolution so + // only compare against confident module resolution. + errs.push(match resolve_module_confident(db, from_list.name(db)) { None => ModuleInconsistency { db, from_list, From 8ebecb2a88d664a6291af71805b8f6876f414815 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Dec 2025 20:42:21 +0000 Subject: [PATCH 07/41] [ty] Add subdiagnostic hint if the user wrote `X = Any` rather than `X: Any` (#21777) --- .../diagnostics/special_form_attributes.md | 26 ++++ ...iagnostics_for_inva…_(249d635e74a41c9e).snap | 114 ++++++++++++++++++ .../src/types/infer/builder.rs | 45 +++++++ 3 files changed, 185 insertions(+) create mode 100644 crates/ty_python_semantic/resources/mdtest/diagnostics/special_form_attributes.md create mode 100644 crates/ty_python_semantic/resources/mdtest/snapshots/special_form_attribu…_-_Diagnostics_for_inva…_(249d635e74a41c9e).snap diff --git a/crates/ty_python_semantic/resources/mdtest/diagnostics/special_form_attributes.md b/crates/ty_python_semantic/resources/mdtest/diagnostics/special_form_attributes.md new file mode 100644 index 0000000000..d19d2a8c12 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/diagnostics/special_form_attributes.md @@ -0,0 +1,26 @@ +# Diagnostics for invalid attribute access on special forms + + + +```py +from typing_extensions import Any, Final, LiteralString, Self + +X = Any + +class Foo: + X: Final = LiteralString + a: int + b: Self + + class Bar: + def __init__(self): + self.y: Final = LiteralString + +X.foo # error: [unresolved-attribute] +X.aaaaooooooo # error: [unresolved-attribute] +Foo.X.startswith # error: [unresolved-attribute] +Foo.Bar().y.startswith # error: [unresolved-attribute] + +# TODO: false positive (just testing the diagnostic in the meantime) +Foo().b.a # error: [unresolved-attribute] +``` diff --git a/crates/ty_python_semantic/resources/mdtest/snapshots/special_form_attribu…_-_Diagnostics_for_inva…_(249d635e74a41c9e).snap b/crates/ty_python_semantic/resources/mdtest/snapshots/special_form_attribu…_-_Diagnostics_for_inva…_(249d635e74a41c9e).snap new file mode 100644 index 0000000000..8672225ce0 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/snapshots/special_form_attribu…_-_Diagnostics_for_inva…_(249d635e74a41c9e).snap @@ -0,0 +1,114 @@ +--- +source: crates/ty_test/src/lib.rs +expression: snapshot +--- +--- +mdtest name: special_form_attributes.md - Diagnostics for invalid attribute access on special forms +mdtest path: crates/ty_python_semantic/resources/mdtest/diagnostics/special_form_attributes.md +--- + +# Python source files + +## mdtest_snippet.py + +``` + 1 | from typing_extensions import Any, Final, LiteralString, Self + 2 | + 3 | X = Any + 4 | + 5 | class Foo: + 6 | X: Final = LiteralString + 7 | a: int + 8 | b: Self + 9 | +10 | class Bar: +11 | def __init__(self): +12 | self.y: Final = LiteralString +13 | +14 | X.foo # error: [unresolved-attribute] +15 | X.aaaaooooooo # error: [unresolved-attribute] +16 | Foo.X.startswith # error: [unresolved-attribute] +17 | Foo.Bar().y.startswith # error: [unresolved-attribute] +18 | +19 | # TODO: false positive (just testing the diagnostic in the meantime) +20 | Foo().b.a # error: [unresolved-attribute] +``` + +# Diagnostics + +``` +error[unresolved-attribute]: Special form `typing.Any` has no attribute `foo` + --> src/mdtest_snippet.py:14:1 + | +12 | self.y: Final = LiteralString +13 | +14 | X.foo # error: [unresolved-attribute] + | ^^^^^ +15 | X.aaaaooooooo # error: [unresolved-attribute] +16 | Foo.X.startswith # error: [unresolved-attribute] + | +help: Objects with type `Any` have a `foo` attribute, but the symbol `typing.Any` does not itself inhabit the type `Any` +help: This error may indicate that `X` was defined as `X = typing.Any` when `X: typing.Any` was intended +info: rule `unresolved-attribute` is enabled by default + +``` + +``` +error[unresolved-attribute]: Special form `typing.Any` has no attribute `aaaaooooooo` + --> src/mdtest_snippet.py:15:1 + | +14 | X.foo # error: [unresolved-attribute] +15 | X.aaaaooooooo # error: [unresolved-attribute] + | ^^^^^^^^^^^^^ +16 | Foo.X.startswith # error: [unresolved-attribute] +17 | Foo.Bar().y.startswith # error: [unresolved-attribute] + | +help: Objects with type `Any` have an `aaaaooooooo` attribute, but the symbol `typing.Any` does not itself inhabit the type `Any` +help: This error may indicate that `X` was defined as `X = typing.Any` when `X: typing.Any` was intended +info: rule `unresolved-attribute` is enabled by default + +``` + +``` +error[unresolved-attribute]: Special form `typing.LiteralString` has no attribute `startswith` + --> src/mdtest_snippet.py:16:1 + | +14 | X.foo # error: [unresolved-attribute] +15 | X.aaaaooooooo # error: [unresolved-attribute] +16 | Foo.X.startswith # error: [unresolved-attribute] + | ^^^^^^^^^^^^^^^^ +17 | Foo.Bar().y.startswith # error: [unresolved-attribute] + | +help: Objects with type `LiteralString` have a `startswith` attribute, but the symbol `typing.LiteralString` does not itself inhabit the type `LiteralString` +help: This error may indicate that `Foo.X` was defined as `Foo.X = typing.LiteralString` when `Foo.X: typing.LiteralString` was intended +info: rule `unresolved-attribute` is enabled by default + +``` + +``` +error[unresolved-attribute]: Special form `typing.LiteralString` has no attribute `startswith` + --> src/mdtest_snippet.py:17:1 + | +15 | X.aaaaooooooo # error: [unresolved-attribute] +16 | Foo.X.startswith # error: [unresolved-attribute] +17 | Foo.Bar().y.startswith # error: [unresolved-attribute] + | ^^^^^^^^^^^^^^^^^^^^^^ +18 | +19 | # TODO: false positive (just testing the diagnostic in the meantime) + | +help: Objects with type `LiteralString` have a `startswith` attribute, but the symbol `typing.LiteralString` does not itself inhabit the type `LiteralString` +info: rule `unresolved-attribute` is enabled by default + +``` + +``` +error[unresolved-attribute]: Special form `typing.Self` has no attribute `a` + --> src/mdtest_snippet.py:20:1 + | +19 | # TODO: false positive (just testing the diagnostic in the meantime) +20 | Foo().b.a # error: [unresolved-attribute] + | ^^^^^^^^^ + | +info: rule `unresolved-attribute` is enabled by default + +``` diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index d45b35298d..9488d2270d 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -4,6 +4,7 @@ use itertools::{Either, EitherOrBoth, Itertools}; use ruff_db::diagnostic::{Annotation, Diagnostic, DiagnosticId, Severity, Span}; use ruff_db::files::File; use ruff_db::parsed::{ParsedModuleRef, parsed_module}; +use ruff_db::source::source_text; use ruff_python_ast::visitor::{Visitor, walk_expr}; use ruff_python_ast::{ self as ast, AnyNodeRef, ExprContext, HasNodeIndex, NodeIndex, PythonVersion, @@ -9111,6 +9112,14 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { /// Infer the type of a [`ast::ExprAttribute`] expression, assuming a load context. fn infer_attribute_load(&mut self, attribute: &ast::ExprAttribute) -> Type<'db> { + fn is_dotted_name(attribute: &ast::Expr) -> bool { + match attribute { + ast::Expr::Name(_) => true, + ast::Expr::Attribute(ast::ExprAttribute { value, .. }) => is_dotted_name(value), + _ => false, + } + } + let ast::ExprAttribute { value, attr, .. } = attribute; let value_type = self.infer_maybe_standalone_expression(value, TypeContext::default()); @@ -9204,6 +9213,42 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } + if let Type::SpecialForm(special_form) = value_type { + if let Some(builder) = + self.context.report_lint(&UNRESOLVED_ATTRIBUTE, attribute) + { + let mut diag = builder.into_diagnostic(format_args!( + "Special form `{special_form}` has no attribute `{attr_name}`", + )); + if let Ok(defined_type) = value_type.in_type_expression( + db, + self.scope(), + self.typevar_binding_context, + ) && !defined_type.member(db, attr_name).place.is_undefined() + { + diag.help(format_args!( + "Objects with type `{ty}` have a{maybe_n} `{attr_name}` attribute, but the symbol \ + `{special_form}` does not itself inhabit the type `{ty}`", + maybe_n = if attr_name.starts_with(['a', 'e', 'i', 'o', 'u']) { + "n" + } else { + "" + }, + ty = defined_type.display(self.db()) + )); + if is_dotted_name(value) { + let source = &source_text(self.db(), self.file())[value.range()]; + diag.help(format_args!( + "This error may indicate that `{source}` was defined as \ + `{source} = {special_form}` when `{source}: {special_form}` \ + was intended" + )); + } + } + } + return fallback(); + } + let Some(builder) = self.context.report_lint(&UNRESOLVED_ATTRIBUTE, attribute) else { return fallback(); From 14fce0d44003ea30c30eb555344bd319da511380 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Dec 2025 21:19:59 +0000 Subject: [PATCH 08/41] [ty] Improve the display of various special-form types (#21775) --- crates/ty_ide/src/inlay_hints.rs | 6 +- .../resources/mdtest/annotations/any.md | 4 +- .../resources/mdtest/annotations/never.md | 2 +- .../resources/mdtest/binary/classes.md | 12 +- .../resources/mdtest/class/super.md | 6 +- .../resources/mdtest/function/return_type.md | 2 +- .../mdtest/generics/pep695/aliases.md | 32 +-- .../resources/mdtest/implicit_type_aliases.md | 195 +++++++++--------- .../resources/mdtest/import/conventions.md | 8 +- .../resources/mdtest/import/star.md | 2 +- .../resources/mdtest/mro.md | 2 +- .../resources/mdtest/narrow/isinstance.md | 2 +- .../resources/mdtest/narrow/issubclass.md | 2 +- .../resources/mdtest/pep613_type_aliases.md | 4 +- ...classinfo`_is_an_in…_(eeef56c0ef87a30b).snap | 4 +- ...nresolvable_MROs_in…_(e2b355c09a967862).snap | 2 +- ...ls_to_protocol_cl…_(288988036f34ddcf).snap | 2 +- crates/ty_python_semantic/src/types.rs | 27 ++- .../src/types/diagnostic.rs | 10 +- .../ty_python_semantic/src/types/display.rs | 74 +++++-- 20 files changed, 220 insertions(+), 178 deletions(-) diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index fea9b2030f..c6ddf8d846 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -6428,11 +6428,11 @@ mod tests { a = Literal['a', 'b', 'c']", ); - assert_snapshot!(test.inlay_hints(), @r" + assert_snapshot!(test.inlay_hints(), @r#" from typing import Literal - a[: ] = Literal['a', 'b', 'c'] - "); + a[: ] = Literal['a', 'b', 'c'] + "#); } struct InlayHintLocationDiagnostic { diff --git a/crates/ty_python_semantic/resources/mdtest/annotations/any.md b/crates/ty_python_semantic/resources/mdtest/annotations/any.md index c2cc2d2461..1bdc306112 100644 --- a/crates/ty_python_semantic/resources/mdtest/annotations/any.md +++ b/crates/ty_python_semantic/resources/mdtest/annotations/any.md @@ -169,13 +169,13 @@ def f(x: Any[int]): `Any` cannot be called (this leads to a `TypeError` at runtime): ```py -Any() # error: [call-non-callable] "Object of type `typing.Any` is not callable" +Any() # error: [call-non-callable] "Object of type `` is not callable" ``` `Any` also cannot be used as a metaclass (under the hood, this leads to an implicit call to `Any`): ```py -class F(metaclass=Any): ... # error: [invalid-metaclass] "Metaclass type `typing.Any` is not callable" +class F(metaclass=Any): ... # error: [invalid-metaclass] "Metaclass type `` is not callable" ``` And `Any` cannot be used in `isinstance()` checks: diff --git a/crates/ty_python_semantic/resources/mdtest/annotations/never.md b/crates/ty_python_semantic/resources/mdtest/annotations/never.md index 81efd2d864..2cd7845771 100644 --- a/crates/ty_python_semantic/resources/mdtest/annotations/never.md +++ b/crates/ty_python_semantic/resources/mdtest/annotations/never.md @@ -59,7 +59,7 @@ python-version = "3.11" ```py from typing import Never -reveal_type(Never) # revealed: typing.Never +reveal_type(Never) # revealed: ``` ### Python 3.10 diff --git a/crates/ty_python_semantic/resources/mdtest/binary/classes.md b/crates/ty_python_semantic/resources/mdtest/binary/classes.md index db42286c84..4a3580a8de 100644 --- a/crates/ty_python_semantic/resources/mdtest/binary/classes.md +++ b/crates/ty_python_semantic/resources/mdtest/binary/classes.md @@ -13,7 +13,7 @@ python-version = "3.10" class A: ... class B: ... -reveal_type(A | B) # revealed: types.UnionType +reveal_type(A | B) # revealed: ``` ## Union of two classes (prior to 3.10) @@ -43,14 +43,14 @@ class A: ... class B: ... def _(sub_a: type[A], sub_b: type[B]): - reveal_type(A | sub_b) # revealed: types.UnionType - reveal_type(sub_a | B) # revealed: types.UnionType - reveal_type(sub_a | sub_b) # revealed: types.UnionType + reveal_type(A | sub_b) # revealed: + reveal_type(sub_a | B) # revealed: + reveal_type(sub_a | sub_b) # revealed: class C[T]: ... class D[T]: ... -reveal_type(C | D) # revealed: types.UnionType +reveal_type(C | D) # revealed: -reveal_type(C[int] | D[str]) # revealed: types.UnionType +reveal_type(C[int] | D[str]) # revealed: ``` diff --git a/crates/ty_python_semantic/resources/mdtest/class/super.md b/crates/ty_python_semantic/resources/mdtest/class/super.md index e4dd9b77bc..750f589125 100644 --- a/crates/ty_python_semantic/resources/mdtest/class/super.md +++ b/crates/ty_python_semantic/resources/mdtest/class/super.md @@ -603,12 +603,14 @@ super(object, object()).__class__ # Not all objects valid in a class's bases list are valid as the first argument to `super()`. # For example, it's valid to inherit from `typing.ChainMap`, but it's not valid as the first argument to `super()`. # -# error: [invalid-super-argument] "`typing.ChainMap` is not a valid class" +# error: [invalid-super-argument] "`` is not a valid class" reveal_type(super(typing.ChainMap, collections.ChainMap())) # revealed: Unknown # Meanwhile, it's not valid to inherit from unsubscripted `typing.Generic`, # but it *is* valid as the first argument to `super()`. -reveal_type(super(typing.Generic, typing.SupportsInt)) # revealed: > +# +# revealed: , > +reveal_type(super(typing.Generic, typing.SupportsInt)) def _(x: type[typing.Any], y: typing.Any): reveal_type(super(x, y)) # revealed: diff --git a/crates/ty_python_semantic/resources/mdtest/function/return_type.md b/crates/ty_python_semantic/resources/mdtest/function/return_type.md index b985ddfe3d..81ccb339e7 100644 --- a/crates/ty_python_semantic/resources/mdtest/function/return_type.md +++ b/crates/ty_python_semantic/resources/mdtest/function/return_type.md @@ -80,7 +80,7 @@ class Foo(Protocol): def f[T](self, v: T) -> T: ... t = (Protocol, int) -reveal_type(t[0]) # revealed: typing.Protocol +reveal_type(t[0]) # revealed: class Lorem(t[0]): def f(self) -> int: ... diff --git a/crates/ty_python_semantic/resources/mdtest/generics/pep695/aliases.md b/crates/ty_python_semantic/resources/mdtest/generics/pep695/aliases.md index 57fc838498..e10febeaeb 100644 --- a/crates/ty_python_semantic/resources/mdtest/generics/pep695/aliases.md +++ b/crates/ty_python_semantic/resources/mdtest/generics/pep695/aliases.md @@ -62,7 +62,7 @@ The specialization must match the generic types: ```py # error: [invalid-type-arguments] "Too many type arguments: expected 1, got 2" -reveal_type(C[int, int]) # revealed: C[Unknown] +reveal_type(C[int, int]) # revealed: ``` And non-generic types cannot be specialized: @@ -85,19 +85,19 @@ type BoundedByUnion[T: int | str] = ... class IntSubclass(int): ... -reveal_type(Bounded[int]) # revealed: Bounded[int] -reveal_type(Bounded[IntSubclass]) # revealed: Bounded[IntSubclass] +reveal_type(Bounded[int]) # revealed: +reveal_type(Bounded[IntSubclass]) # revealed: # error: [invalid-type-arguments] "Type `str` is not assignable to upper bound `int` of type variable `T@Bounded`" -reveal_type(Bounded[str]) # revealed: Bounded[Unknown] +reveal_type(Bounded[str]) # revealed: # error: [invalid-type-arguments] "Type `int | str` is not assignable to upper bound `int` of type variable `T@Bounded`" -reveal_type(Bounded[int | str]) # revealed: Bounded[Unknown] +reveal_type(Bounded[int | str]) # revealed: -reveal_type(BoundedByUnion[int]) # revealed: BoundedByUnion[int] -reveal_type(BoundedByUnion[IntSubclass]) # revealed: BoundedByUnion[IntSubclass] -reveal_type(BoundedByUnion[str]) # revealed: BoundedByUnion[str] -reveal_type(BoundedByUnion[int | str]) # revealed: BoundedByUnion[int | str] +reveal_type(BoundedByUnion[int]) # revealed: +reveal_type(BoundedByUnion[IntSubclass]) # revealed: +reveal_type(BoundedByUnion[str]) # revealed: +reveal_type(BoundedByUnion[int | str]) # revealed: ``` If the type variable is constrained, the specialized type must satisfy those constraints: @@ -105,20 +105,20 @@ If the type variable is constrained, the specialized type must satisfy those con ```py type Constrained[T: (int, str)] = ... -reveal_type(Constrained[int]) # revealed: Constrained[int] +reveal_type(Constrained[int]) # revealed: # TODO: error: [invalid-argument-type] # TODO: revealed: Constrained[Unknown] -reveal_type(Constrained[IntSubclass]) # revealed: Constrained[IntSubclass] +reveal_type(Constrained[IntSubclass]) # revealed: -reveal_type(Constrained[str]) # revealed: Constrained[str] +reveal_type(Constrained[str]) # revealed: # TODO: error: [invalid-argument-type] # TODO: revealed: Unknown -reveal_type(Constrained[int | str]) # revealed: Constrained[int | str] +reveal_type(Constrained[int | str]) # revealed: # error: [invalid-type-arguments] "Type `object` does not satisfy constraints `int`, `str` of type variable `T@Constrained`" -reveal_type(Constrained[object]) # revealed: Constrained[Unknown] +reveal_type(Constrained[object]) # revealed: ``` If the type variable has a default, it can be omitted: @@ -126,8 +126,8 @@ If the type variable has a default, it can be omitted: ```py type WithDefault[T, U = int] = ... -reveal_type(WithDefault[str, str]) # revealed: WithDefault[str, str] -reveal_type(WithDefault[str]) # revealed: WithDefault[str, int] +reveal_type(WithDefault[str, str]) # revealed: +reveal_type(WithDefault[str]) # revealed: ``` If the type alias is not specialized explicitly, it is implicitly specialized to `Unknown`: diff --git a/crates/ty_python_semantic/resources/mdtest/implicit_type_aliases.md b/crates/ty_python_semantic/resources/mdtest/implicit_type_aliases.md index ed73c9323b..d0836559f3 100644 --- a/crates/ty_python_semantic/resources/mdtest/implicit_type_aliases.md +++ b/crates/ty_python_semantic/resources/mdtest/implicit_type_aliases.md @@ -77,44 +77,44 @@ IntOrTypeVar = int | T TypeVarOrNone = T | None NoneOrTypeVar = None | T -reveal_type(IntOrStr) # revealed: types.UnionType -reveal_type(IntOrStrOrBytes1) # revealed: types.UnionType -reveal_type(IntOrStrOrBytes2) # revealed: types.UnionType -reveal_type(IntOrStrOrBytes3) # revealed: types.UnionType -reveal_type(IntOrStrOrBytes4) # revealed: types.UnionType -reveal_type(IntOrStrOrBytes5) # revealed: types.UnionType -reveal_type(IntOrStrOrBytes6) # revealed: types.UnionType -reveal_type(BytesOrIntOrStr) # revealed: types.UnionType -reveal_type(IntOrNone) # revealed: types.UnionType -reveal_type(NoneOrInt) # revealed: types.UnionType -reveal_type(IntOrStrOrNone) # revealed: types.UnionType -reveal_type(NoneOrIntOrStr) # revealed: types.UnionType -reveal_type(IntOrAny) # revealed: types.UnionType -reveal_type(AnyOrInt) # revealed: types.UnionType -reveal_type(NoneOrAny) # revealed: types.UnionType -reveal_type(AnyOrNone) # revealed: types.UnionType -reveal_type(NeverOrAny) # revealed: types.UnionType -reveal_type(AnyOrNever) # revealed: types.UnionType -reveal_type(UnknownOrInt) # revealed: types.UnionType -reveal_type(IntOrUnknown) # revealed: types.UnionType -reveal_type(StrOrZero) # revealed: types.UnionType -reveal_type(ZeroOrStr) # revealed: types.UnionType -reveal_type(IntOrLiteralString) # revealed: types.UnionType -reveal_type(LiteralStringOrInt) # revealed: types.UnionType -reveal_type(NoneOrTuple) # revealed: types.UnionType -reveal_type(TupleOrNone) # revealed: types.UnionType -reveal_type(IntOrAnnotated) # revealed: types.UnionType -reveal_type(AnnotatedOrInt) # revealed: types.UnionType -reveal_type(IntOrOptional) # revealed: types.UnionType -reveal_type(OptionalOrInt) # revealed: types.UnionType -reveal_type(IntOrTypeOfStr) # revealed: types.UnionType -reveal_type(TypeOfStrOrInt) # revealed: types.UnionType -reveal_type(IntOrCallable) # revealed: types.UnionType -reveal_type(CallableOrInt) # revealed: types.UnionType -reveal_type(TypeVarOrInt) # revealed: types.UnionType -reveal_type(IntOrTypeVar) # revealed: types.UnionType -reveal_type(TypeVarOrNone) # revealed: types.UnionType -reveal_type(NoneOrTypeVar) # revealed: types.UnionType +reveal_type(IntOrStr) # revealed: +reveal_type(IntOrStrOrBytes1) # revealed: +reveal_type(IntOrStrOrBytes2) # revealed: +reveal_type(IntOrStrOrBytes3) # revealed: +reveal_type(IntOrStrOrBytes4) # revealed: +reveal_type(IntOrStrOrBytes5) # revealed: +reveal_type(IntOrStrOrBytes6) # revealed: +reveal_type(BytesOrIntOrStr) # revealed: +reveal_type(IntOrNone) # revealed: +reveal_type(NoneOrInt) # revealed: +reveal_type(IntOrStrOrNone) # revealed: +reveal_type(NoneOrIntOrStr) # revealed: +reveal_type(IntOrAny) # revealed: +reveal_type(AnyOrInt) # revealed: +reveal_type(NoneOrAny) # revealed: +reveal_type(AnyOrNone) # revealed: +reveal_type(NeverOrAny) # revealed: +reveal_type(AnyOrNever) # revealed: +reveal_type(UnknownOrInt) # revealed: +reveal_type(IntOrUnknown) # revealed: +reveal_type(StrOrZero) # revealed: +reveal_type(ZeroOrStr) # revealed: +reveal_type(IntOrLiteralString) # revealed: +reveal_type(LiteralStringOrInt) # revealed: +reveal_type(NoneOrTuple) # revealed: +reveal_type(TupleOrNone) # revealed: +reveal_type(IntOrAnnotated) # revealed: +reveal_type(AnnotatedOrInt) # revealed: +reveal_type(IntOrOptional) # revealed: +reveal_type(OptionalOrInt) # revealed: +reveal_type(IntOrTypeOfStr) # revealed: +reveal_type(TypeOfStrOrInt) # revealed: +reveal_type(IntOrCallable) # revealed: bytes)'> +reveal_type(CallableOrInt) # revealed: bytes) | int'> +reveal_type(TypeVarOrInt) # revealed: +reveal_type(IntOrTypeVar) # revealed: +reveal_type(TypeVarOrNone) # revealed: +reveal_type(NoneOrTypeVar) # revealed: def _( int_or_str: IntOrStr, @@ -295,7 +295,7 @@ X = Foo | Bar # In an ideal world, perhaps we would respect `Meta.__or__` here and reveal `str`? # But we still need to record what the elements are, since (according to the typing spec) # `X` is still a valid type alias -reveal_type(X) # revealed: types.UnionType +reveal_type(X) # revealed: def f(obj: X): reveal_type(obj) # revealed: Foo | Bar @@ -391,16 +391,17 @@ MyOptional = T | None reveal_type(MyList) # revealed: reveal_type(MyDict) # revealed: -reveal_type(MyType) # revealed: GenericAlias +reveal_type(MyType) # revealed: reveal_type(IntAndType) # revealed: reveal_type(Pair) # revealed: reveal_type(Sum) # revealed: -reveal_type(ListOrTuple) # revealed: types.UnionType -reveal_type(ListOrTupleLegacy) # revealed: types.UnionType +reveal_type(ListOrTuple) # revealed: +# revealed: +reveal_type(ListOrTupleLegacy) reveal_type(MyCallable) # revealed: @Todo(Callable[..] specialized with ParamSpec) -reveal_type(AnnotatedType) # revealed: +reveal_type(AnnotatedType) # revealed: ]'> reveal_type(TransparentAlias) # revealed: typing.TypeVar -reveal_type(MyOptional) # revealed: types.UnionType +reveal_type(MyOptional) # revealed: def _( list_of_ints: MyList[int], @@ -456,12 +457,12 @@ AnnotatedInt = AnnotatedType[int] SubclassOfInt = MyType[int] CallableIntToStr = MyCallable[[int], str] -reveal_type(IntsOrNone) # revealed: types.UnionType -reveal_type(IntsOrStrs) # revealed: types.UnionType +reveal_type(IntsOrNone) # revealed: +reveal_type(IntsOrStrs) # revealed: reveal_type(ListOfPairs) # revealed: -reveal_type(ListOrTupleOfInts) # revealed: types.UnionType -reveal_type(AnnotatedInt) # revealed: -reveal_type(SubclassOfInt) # revealed: GenericAlias +reveal_type(ListOrTupleOfInts) # revealed: +reveal_type(AnnotatedInt) # revealed: ]'> +reveal_type(SubclassOfInt) # revealed: reveal_type(CallableIntToStr) # revealed: @Todo(Callable[..] specialized with ParamSpec) def _( @@ -495,8 +496,8 @@ MyOtherType = MyType[T] TypeOrList = MyType[B] | MyList[B] reveal_type(MyOtherList) # revealed: -reveal_type(MyOtherType) # revealed: GenericAlias -reveal_type(TypeOrList) # revealed: types.UnionType +reveal_type(MyOtherType) # revealed: +reveal_type(TypeOrList) # revealed: def _( list_of_ints: MyOtherList[int], @@ -898,7 +899,7 @@ from typing import Optional MyOptionalInt = Optional[int] -reveal_type(MyOptionalInt) # revealed: types.UnionType +reveal_type(MyOptionalInt) # revealed: def _(optional_int: MyOptionalInt): reveal_type(optional_int) # revealed: int | None @@ -931,9 +932,9 @@ MyLiteralString = LiteralString MyNoReturn = NoReturn MyNever = Never -reveal_type(MyLiteralString) # revealed: typing.LiteralString -reveal_type(MyNoReturn) # revealed: typing.NoReturn -reveal_type(MyNever) # revealed: typing.Never +reveal_type(MyLiteralString) # revealed: +reveal_type(MyNoReturn) # revealed: +reveal_type(MyNever) # revealed: def _( ls: MyLiteralString, @@ -986,8 +987,8 @@ from typing import Union IntOrStr = Union[int, str] IntOrStrOrBytes = Union[int, Union[str, bytes]] -reveal_type(IntOrStr) # revealed: types.UnionType -reveal_type(IntOrStrOrBytes) # revealed: types.UnionType +reveal_type(IntOrStr) # revealed: +reveal_type(IntOrStrOrBytes) # revealed: def _( int_or_str: IntOrStr, @@ -1015,7 +1016,7 @@ An empty `typing.Union` leads to a `TypeError` at runtime, so we emit an error. # error: [invalid-type-form] "`typing.Union` requires at least one type argument" EmptyUnion = Union[()] -reveal_type(EmptyUnion) # revealed: types.UnionType +reveal_type(EmptyUnion) # revealed: def _(empty: EmptyUnion): reveal_type(empty) # revealed: Never @@ -1060,14 +1061,14 @@ SubclassOfG = type[G] SubclassOfGInt = type[G[int]] SubclassOfP = type[P] -reveal_type(SubclassOfA) # revealed: GenericAlias -reveal_type(SubclassOfAny) # revealed: GenericAlias -reveal_type(SubclassOfAOrB1) # revealed: GenericAlias -reveal_type(SubclassOfAOrB2) # revealed: types.UnionType -reveal_type(SubclassOfAOrB3) # revealed: types.UnionType -reveal_type(SubclassOfG) # revealed: GenericAlias -reveal_type(SubclassOfGInt) # revealed: GenericAlias -reveal_type(SubclassOfP) # revealed: GenericAlias +reveal_type(SubclassOfA) # revealed: +reveal_type(SubclassOfAny) # revealed: +reveal_type(SubclassOfAOrB1) # revealed: +reveal_type(SubclassOfAOrB2) # revealed: +reveal_type(SubclassOfAOrB3) # revealed: +reveal_type(SubclassOfG) # revealed: +reveal_type(SubclassOfGInt) # revealed: +reveal_type(SubclassOfP) # revealed: def _( subclass_of_a: SubclassOfA, @@ -1148,14 +1149,14 @@ SubclassOfG = Type[G] SubclassOfGInt = Type[G[int]] SubclassOfP = Type[P] -reveal_type(SubclassOfA) # revealed: GenericAlias -reveal_type(SubclassOfAny) # revealed: GenericAlias -reveal_type(SubclassOfAOrB1) # revealed: GenericAlias -reveal_type(SubclassOfAOrB2) # revealed: types.UnionType -reveal_type(SubclassOfAOrB3) # revealed: types.UnionType -reveal_type(SubclassOfG) # revealed: GenericAlias -reveal_type(SubclassOfGInt) # revealed: GenericAlias -reveal_type(SubclassOfP) # revealed: GenericAlias +reveal_type(SubclassOfA) # revealed: +reveal_type(SubclassOfAny) # revealed: +reveal_type(SubclassOfAOrB1) # revealed: +reveal_type(SubclassOfAOrB2) # revealed: +reveal_type(SubclassOfAOrB3) # revealed: +reveal_type(SubclassOfG) # revealed: +reveal_type(SubclassOfGInt) # revealed: +reveal_type(SubclassOfP) # revealed: def _( subclass_of_a: SubclassOfA, @@ -1270,25 +1271,25 @@ DefaultDictOrNone = DefaultDict[str, int] | None DequeOrNone = Deque[str] | None OrderedDictOrNone = OrderedDict[str, int] | None -reveal_type(NoneOrList) # revealed: types.UnionType -reveal_type(NoneOrSet) # revealed: types.UnionType -reveal_type(NoneOrDict) # revealed: types.UnionType -reveal_type(NoneOrFrozenSet) # revealed: types.UnionType -reveal_type(NoneOrChainMap) # revealed: types.UnionType -reveal_type(NoneOrCounter) # revealed: types.UnionType -reveal_type(NoneOrDefaultDict) # revealed: types.UnionType -reveal_type(NoneOrDeque) # revealed: types.UnionType -reveal_type(NoneOrOrderedDict) # revealed: types.UnionType +reveal_type(NoneOrList) # revealed: +reveal_type(NoneOrSet) # revealed: +reveal_type(NoneOrDict) # revealed: +reveal_type(NoneOrFrozenSet) # revealed: +reveal_type(NoneOrChainMap) # revealed: +reveal_type(NoneOrCounter) # revealed: +reveal_type(NoneOrDefaultDict) # revealed: +reveal_type(NoneOrDeque) # revealed: +reveal_type(NoneOrOrderedDict) # revealed: -reveal_type(ListOrNone) # revealed: types.UnionType -reveal_type(SetOrNone) # revealed: types.UnionType -reveal_type(DictOrNone) # revealed: types.UnionType -reveal_type(FrozenSetOrNone) # revealed: types.UnionType -reveal_type(ChainMapOrNone) # revealed: types.UnionType -reveal_type(CounterOrNone) # revealed: types.UnionType -reveal_type(DefaultDictOrNone) # revealed: types.UnionType -reveal_type(DequeOrNone) # revealed: types.UnionType -reveal_type(OrderedDictOrNone) # revealed: types.UnionType +reveal_type(ListOrNone) # revealed: +reveal_type(SetOrNone) # revealed: +reveal_type(DictOrNone) # revealed: +reveal_type(FrozenSetOrNone) # revealed: +reveal_type(ChainMapOrNone) # revealed: +reveal_type(CounterOrNone) # revealed: +reveal_type(DefaultDictOrNone) # revealed: +reveal_type(DequeOrNone) # revealed: +reveal_type(OrderedDictOrNone) # revealed: def _( none_or_list: NoneOrList, @@ -1381,9 +1382,9 @@ CallableNoArgs = Callable[[], None] BasicCallable = Callable[[int, str], bytes] GradualCallable = Callable[..., str] -reveal_type(CallableNoArgs) # revealed: GenericAlias -reveal_type(BasicCallable) # revealed: GenericAlias -reveal_type(GradualCallable) # revealed: GenericAlias +reveal_type(CallableNoArgs) # revealed: None'> +reveal_type(BasicCallable) # revealed: bytes'> +reveal_type(GradualCallable) # revealed: str'> def _( callable_no_args: CallableNoArgs, @@ -1415,8 +1416,8 @@ InvalidCallable1 = Callable[[int]] # error: [invalid-type-form] "The first argument to `Callable` must be either a list of types, ParamSpec, Concatenate, or `...`" InvalidCallable2 = Callable[int, str] -reveal_type(InvalidCallable1) # revealed: GenericAlias -reveal_type(InvalidCallable2) # revealed: GenericAlias +reveal_type(InvalidCallable1) # revealed: Unknown'> +reveal_type(InvalidCallable2) # revealed: Unknown'> def _(invalid_callable1: InvalidCallable1, invalid_callable2: InvalidCallable2): reveal_type(invalid_callable1) # revealed: (...) -> Unknown diff --git a/crates/ty_python_semantic/resources/mdtest/import/conventions.md b/crates/ty_python_semantic/resources/mdtest/import/conventions.md index 48d93515a8..28ea0b4bc2 100644 --- a/crates/ty_python_semantic/resources/mdtest/import/conventions.md +++ b/crates/ty_python_semantic/resources/mdtest/import/conventions.md @@ -53,8 +53,8 @@ in `import os.path as os.path` the `os.path` is not a valid identifier. ```py from b import Any, Literal, foo -reveal_type(Any) # revealed: typing.Any -reveal_type(Literal) # revealed: typing.Literal +reveal_type(Any) # revealed: +reveal_type(Literal) # revealed: reveal_type(foo) # revealed: ``` @@ -132,7 +132,7 @@ reveal_type(Any) # revealed: Unknown ```pyi from typing import Any -reveal_type(Any) # revealed: typing.Any +reveal_type(Any) # revealed: ``` ## Nested mixed re-export and not @@ -169,7 +169,7 @@ reveal_type(Any) # revealed: Unknown ```pyi from typing import Any -reveal_type(Any) # revealed: typing.Any +reveal_type(Any) # revealed: ``` ## Exported as different name diff --git a/crates/ty_python_semantic/resources/mdtest/import/star.md b/crates/ty_python_semantic/resources/mdtest/import/star.md index 14cff45efc..a60c0062b1 100644 --- a/crates/ty_python_semantic/resources/mdtest/import/star.md +++ b/crates/ty_python_semantic/resources/mdtest/import/star.md @@ -1437,7 +1437,7 @@ are present due to `*` imports. import collections.abc reveal_type(collections.abc.Sequence) # revealed: -reveal_type(collections.abc.Callable) # revealed: typing.Callable +reveal_type(collections.abc.Callable) # revealed: reveal_type(collections.abc.Set) # revealed: ``` diff --git a/crates/ty_python_semantic/resources/mdtest/mro.md b/crates/ty_python_semantic/resources/mdtest/mro.md index b158cbb62b..da57d7f9a7 100644 --- a/crates/ty_python_semantic/resources/mdtest/mro.md +++ b/crates/ty_python_semantic/resources/mdtest/mro.md @@ -301,7 +301,7 @@ class B: ... EitherOr = A | B -# error: [invalid-base] "Invalid class base with type `types.UnionType`" +# error: [invalid-base] "Invalid class base with type ``" class Foo(EitherOr): ... ``` diff --git a/crates/ty_python_semantic/resources/mdtest/narrow/isinstance.md b/crates/ty_python_semantic/resources/mdtest/narrow/isinstance.md index 6e712b3e71..68562e06ad 100644 --- a/crates/ty_python_semantic/resources/mdtest/narrow/isinstance.md +++ b/crates/ty_python_semantic/resources/mdtest/narrow/isinstance.md @@ -156,7 +156,7 @@ from typing import Union IntOrStr = Union[int, str] -reveal_type(IntOrStr) # revealed: types.UnionType +reveal_type(IntOrStr) # revealed: def _(x: int | str | bytes | memoryview | range): if isinstance(x, IntOrStr): diff --git a/crates/ty_python_semantic/resources/mdtest/narrow/issubclass.md b/crates/ty_python_semantic/resources/mdtest/narrow/issubclass.md index 52fcb6dfb5..ed9964274a 100644 --- a/crates/ty_python_semantic/resources/mdtest/narrow/issubclass.md +++ b/crates/ty_python_semantic/resources/mdtest/narrow/issubclass.md @@ -209,7 +209,7 @@ from typing import Union IntOrStr = Union[int, str] -reveal_type(IntOrStr) # revealed: types.UnionType +reveal_type(IntOrStr) # revealed: def f(x: type[int | str | bytes | range]): if issubclass(x, IntOrStr): diff --git a/crates/ty_python_semantic/resources/mdtest/pep613_type_aliases.md b/crates/ty_python_semantic/resources/mdtest/pep613_type_aliases.md index c41b88b3b1..69f92dda94 100644 --- a/crates/ty_python_semantic/resources/mdtest/pep613_type_aliases.md +++ b/crates/ty_python_semantic/resources/mdtest/pep613_type_aliases.md @@ -113,7 +113,7 @@ MyList: TypeAlias = list[T] ListOrSet: TypeAlias = list[T] | set[T] reveal_type(MyList) # revealed: -reveal_type(ListOrSet) # revealed: types.UnionType +reveal_type(ListOrSet) # revealed: def _(list_of_int: MyList[int], list_or_set_of_str: ListOrSet[str]): reveal_type(list_of_int) # revealed: list[int] @@ -293,7 +293,7 @@ def _(rec: RecursiveHomogeneousTuple): reveal_type(rec) # revealed: tuple[Divergent, ...] ClassInfo: TypeAlias = type | UnionType | tuple["ClassInfo", ...] -reveal_type(ClassInfo) # revealed: types.UnionType +reveal_type(ClassInfo) # revealed: def my_isinstance(obj: object, classinfo: ClassInfo) -> bool: # TODO should be `type | UnionType | tuple[ClassInfo, ...]` diff --git a/crates/ty_python_semantic/resources/mdtest/snapshots/isinstance.md_-_Narrowing_for_`isins…_-_`classinfo`_is_an_in…_(eeef56c0ef87a30b).snap b/crates/ty_python_semantic/resources/mdtest/snapshots/isinstance.md_-_Narrowing_for_`isins…_-_`classinfo`_is_an_in…_(eeef56c0ef87a30b).snap index 822f9319a1..195610b255 100644 --- a/crates/ty_python_semantic/resources/mdtest/snapshots/isinstance.md_-_Narrowing_for_`isins…_-_`classinfo`_is_an_in…_(eeef56c0ef87a30b).snap +++ b/crates/ty_python_semantic/resources/mdtest/snapshots/isinstance.md_-_Narrowing_for_`isins…_-_`classinfo`_is_an_in…_(eeef56c0ef87a30b).snap @@ -63,7 +63,7 @@ error[invalid-argument-type]: Invalid second argument to `isinstance` 10 | # error: [invalid-argument-type] | info: A `UnionType` instance can only be used as the second argument to `isinstance` if all elements are class objects -info: Elements `` and `` in the union are not class objects +info: Elements `` and `` in the union are not class objects info: rule `invalid-argument-type` is enabled by default ``` @@ -82,7 +82,7 @@ error[invalid-argument-type]: Invalid second argument to `isinstance` 13 | else: | info: A `UnionType` instance can only be used as the second argument to `isinstance` if all elements are class objects -info: Element `typing.Any` in the union, and 2 more elements, are not class objects +info: Element `` in the union, and 2 more elements, are not class objects info: rule `invalid-argument-type` is enabled by default ``` diff --git a/crates/ty_python_semantic/resources/mdtest/snapshots/mro.md_-_Method_Resolution_Or…_-_Unresolvable_MROs_in…_(e2b355c09a967862).snap b/crates/ty_python_semantic/resources/mdtest/snapshots/mro.md_-_Method_Resolution_Or…_-_Unresolvable_MROs_in…_(e2b355c09a967862).snap index c7590f4255..42e41b6e87 100644 --- a/crates/ty_python_semantic/resources/mdtest/snapshots/mro.md_-_Method_Resolution_Or…_-_Unresolvable_MROs_in…_(e2b355c09a967862).snap +++ b/crates/ty_python_semantic/resources/mdtest/snapshots/mro.md_-_Method_Resolution_Or…_-_Unresolvable_MROs_in…_(e2b355c09a967862).snap @@ -24,7 +24,7 @@ mdtest path: crates/ty_python_semantic/resources/mdtest/mro.md # Diagnostics ``` -error[inconsistent-mro]: Cannot create a consistent method resolution order (MRO) for class `Baz` with bases list `[typing.Protocol[T], , ]` +error[inconsistent-mro]: Cannot create a consistent method resolution order (MRO) for class `Baz` with bases list `[, , ]` --> src/mdtest_snippet.py:7:1 | 5 | class Foo(Protocol): ... diff --git a/crates/ty_python_semantic/resources/mdtest/snapshots/protocols.md_-_Protocols_-_Calls_to_protocol_cl…_(288988036f34ddcf).snap b/crates/ty_python_semantic/resources/mdtest/snapshots/protocols.md_-_Protocols_-_Calls_to_protocol_cl…_(288988036f34ddcf).snap index 4e18a85fd6..57bedf5797 100644 --- a/crates/ty_python_semantic/resources/mdtest/snapshots/protocols.md_-_Protocols_-_Calls_to_protocol_cl…_(288988036f34ddcf).snap +++ b/crates/ty_python_semantic/resources/mdtest/snapshots/protocols.md_-_Protocols_-_Calls_to_protocol_cl…_(288988036f34ddcf).snap @@ -42,7 +42,7 @@ mdtest path: crates/ty_python_semantic/resources/mdtest/protocols.md # Diagnostics ``` -error[call-non-callable]: Object of type `typing.Protocol` is not callable +error[call-non-callable]: Object of type `` is not callable --> src/mdtest_snippet.py:4:13 | 3 | # error: [call-non-callable] diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index b1cfc9d3c9..81d056f91a 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -7331,7 +7331,7 @@ impl<'db> Type<'db> { | SpecialFormType::Union | SpecialFormType::Intersection => Err(InvalidTypeExpressionError { invalid_expressions: smallvec::smallvec_inline![ - InvalidTypeExpression::RequiresArguments(*self) + InvalidTypeExpression::RequiresArguments(*special_form) ], fallback_type: Type::unknown(), }), @@ -7357,7 +7357,7 @@ impl<'db> Type<'db> { | SpecialFormType::Unpack | SpecialFormType::CallableTypeOf => Err(InvalidTypeExpressionError { invalid_expressions: smallvec::smallvec_inline![ - InvalidTypeExpression::RequiresOneArgument(*self) + InvalidTypeExpression::RequiresOneArgument(*special_form) ], fallback_type: Type::unknown(), }), @@ -7365,7 +7365,7 @@ impl<'db> Type<'db> { SpecialFormType::Annotated | SpecialFormType::Concatenate => { Err(InvalidTypeExpressionError { invalid_expressions: smallvec::smallvec_inline![ - InvalidTypeExpression::RequiresTwoArguments(*self) + InvalidTypeExpression::RequiresTwoArguments(*special_form) ], fallback_type: Type::unknown(), }) @@ -9087,11 +9087,11 @@ impl<'db> InvalidTypeExpressionError<'db> { #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, get_size2::GetSize)] enum InvalidTypeExpression<'db> { /// Some types always require exactly one argument when used in a type expression - RequiresOneArgument(Type<'db>), + RequiresOneArgument(SpecialFormType), /// Some types always require at least one argument when used in a type expression - RequiresArguments(Type<'db>), + RequiresArguments(SpecialFormType), /// Some types always require at least two arguments when used in a type expression - RequiresTwoArguments(Type<'db>), + RequiresTwoArguments(SpecialFormType), /// The `Protocol` class is invalid in type expressions Protocol, /// Same for `Generic` @@ -9131,20 +9131,17 @@ impl<'db> InvalidTypeExpression<'db> { impl std::fmt::Display for Display<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self.error { - InvalidTypeExpression::RequiresOneArgument(ty) => write!( + InvalidTypeExpression::RequiresOneArgument(special_form) => write!( f, - "`{ty}` requires exactly one argument when used in a type expression", - ty = ty.display(self.db) + "`{special_form}` requires exactly one argument when used in a type expression", ), - InvalidTypeExpression::RequiresArguments(ty) => write!( + InvalidTypeExpression::RequiresArguments(special_form) => write!( f, - "`{ty}` requires at least one argument when used in a type expression", - ty = ty.display(self.db) + "`{special_form}` requires at least one argument when used in a type expression", ), - InvalidTypeExpression::RequiresTwoArguments(ty) => write!( + InvalidTypeExpression::RequiresTwoArguments(special_form) => write!( f, - "`{ty}` requires at least two arguments when used in a type expression", - ty = ty.display(self.db) + "`{special_form}` requires at least two arguments when used in a type expression", ), InvalidTypeExpression::Protocol => { f.write_str("`typing.Protocol` is not allowed in type expressions") diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index d72cbf8dbc..2706787998 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -2998,11 +2998,10 @@ pub(crate) fn report_invalid_arguments_to_annotated( let Some(builder) = context.report_lint(&INVALID_TYPE_FORM, subscript) else { return; }; - builder.into_diagnostic(format_args!( - "Special form `{}` expected at least 2 arguments \ + builder.into_diagnostic( + "Special form `typing.Annotated` expected at least 2 arguments \ (one type and at least one metadata element)", - SpecialFormType::Annotated - )); + ); } pub(crate) fn report_invalid_argument_number_to_special_form( @@ -3103,8 +3102,7 @@ pub(crate) fn report_invalid_arguments_to_callable( return; }; builder.into_diagnostic(format_args!( - "Special form `{}` expected exactly two arguments (parameter types and return type)", - SpecialFormType::Callable + "Special form `typing.Callable` expected exactly two arguments (parameter types and return type)", )); } diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index cadd54eef1..95ee3ad748 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -696,7 +696,8 @@ impl<'db> FmtDetailed<'db> for DisplayRepresentation<'db> { ), }, Type::SpecialForm(special_form) => { - write!(f.with_type(self.ty), "{special_form}") + f.set_invalid_syntax(); + write!(f.with_type(self.ty), "") } Type::KnownInstance(known_instance) => known_instance .display_with(self.db, self.settings.clone()) @@ -2173,16 +2174,24 @@ impl<'db> FmtDetailed<'db> for DisplayKnownInstanceRepr<'db> { let ty = Type::KnownInstance(self.known_instance); match self.known_instance { KnownInstanceType::SubscriptedProtocol(generic_context) => { + f.set_invalid_syntax(); + f.write_str("") } KnownInstanceType::SubscriptedGeneric(generic_context) => { + f.set_invalid_syntax(); + f.write_str("") } KnownInstanceType::TypeAliasType(alias) => { if let Some(specialization) = alias.specialization(self.db) { - f.write_str(alias.name(self.db))?; + f.set_invalid_syntax(); + f.write_str(" FmtDetailed<'db> for DisplayKnownInstanceRepr<'db> { DisplaySettings::default(), ) .to_string(), - ) + )?; + f.write_str("'>") } else { f.with_type(ty).write_str("typing.TypeAliasType") } @@ -2201,9 +2211,9 @@ impl<'db> FmtDetailed<'db> for DisplayKnownInstanceRepr<'db> { // have a `Type::TypeVar(_)`, which is rendered as the typevar's name. KnownInstanceType::TypeVar(typevar_instance) => { if typevar_instance.kind(self.db).is_paramspec() { - f.write_str("typing.ParamSpec") + f.with_type(ty).write_str("typing.ParamSpec") } else { - f.write_str("typing.TypeVar") + f.with_type(ty).write_str("typing.TypeVar") } } KnownInstanceType::Deprecated(_) => f.write_str("warnings.deprecated"), @@ -2226,22 +2236,56 @@ impl<'db> FmtDetailed<'db> for DisplayKnownInstanceRepr<'db> { f.with_type(ty).write_str("ty_extensions.Specialization")?; write!(f, "{}", specialization.display_full(self.db)) } - KnownInstanceType::UnionType(_) => f.with_type(ty).write_str("types.UnionType"), - KnownInstanceType::Literal(_) => { + KnownInstanceType::UnionType(union) => { f.set_invalid_syntax(); - f.write_str("") + f.write_char('<')?; + f.with_type(ty).write_str("types.UnionType")?; + f.write_str(" special form")?; + if let Ok(ty) = union.union_type(self.db) { + write!(f, " '{}'", ty.display(self.db))?; + } + f.write_char('>') } - KnownInstanceType::Annotated(_) => { + KnownInstanceType::Literal(inner) => { f.set_invalid_syntax(); - f.write_str("") + write!( + f, + "", + inner.inner(self.db).display(self.db) + ) } - KnownInstanceType::TypeGenericAlias(_) | KnownInstanceType::Callable(_) => { - f.with_type(ty).write_str("GenericAlias") + KnownInstanceType::Annotated(inner) => { + f.set_invalid_syntax(); + f.write_str("]'>", + inner.inner(self.db).display(self.db) + ) + } + KnownInstanceType::Callable(callable) => { + f.set_invalid_syntax(); + f.write_char('<')?; + f.with_type(ty).write_str("typing.Callable")?; + write!(f, " special form '{}'>", callable.display(self.db)) + } + KnownInstanceType::TypeGenericAlias(inner) => { + f.set_invalid_syntax(); + f.write_str("") } KnownInstanceType::LiteralStringAlias(_) => f.write_str("str"), KnownInstanceType::NewType(declaration) => { f.set_invalid_syntax(); - write!(f, "", declaration.name(self.db)) + f.write_str("") } } } From e2b72fbf99920c5621243ee67e1028844a841d35 Mon Sep 17 00:00:00 2001 From: Aria Desires Date: Wed, 3 Dec 2025 16:54:50 -0500 Subject: [PATCH 09/41] [ty] cleanup test path (#21781) Fixes https://github.com/astral-sh/ruff/pull/21745#discussion_r2586552295 --- .../mdtest/import/site_packages_discovery.md | 20 +++++++++---------- .../resources/mdtest/import/workspaces.md | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/ty_python_semantic/resources/mdtest/import/site_packages_discovery.md b/crates/ty_python_semantic/resources/mdtest/import/site_packages_discovery.md index a1c256c375..ebd15d926b 100644 --- a/crates/ty_python_semantic/resources/mdtest/import/site_packages_discovery.md +++ b/crates/ty_python_semantic/resources/mdtest/import/site_packages_discovery.md @@ -22,10 +22,10 @@ python = "/.venv" `/.venv/pyvenv.cfg`: ```cfg -home = /doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin +home = /do/re/mi//cpython-3.13.2-macos-aarch64-none/bin ``` -`/doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin/python`: +`/do/re/mi//cpython-3.13.2-macos-aarch64-none/bin/python`: ```text ``` @@ -54,11 +54,11 @@ python = "/.venv" `/.venv/pyvenv.cfg`: ```cfg -home = /doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin +home = /do/re/mi//cpython-3.13.2-macos-aarch64-none/bin version = wut ``` -`/doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin/python`: +`/do/re/mi//cpython-3.13.2-macos-aarch64-none/bin/python`: ```text ``` @@ -87,11 +87,11 @@ python = "/.venv" `/.venv/pyvenv.cfg`: ```cfg -home = /doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin +home = /do/re/mi//cpython-3.13.2-macos-aarch64-none/bin version_info = no-really-wut ``` -`/doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin/python`: +`/do/re/mi//cpython-3.13.2-macos-aarch64-none/bin/python`: ```text ``` @@ -132,7 +132,7 @@ python = "/.venv" `/.venv/pyvenv.cfg`: ```cfg -home = /doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin +home = /do/re/mi//cpython-3.13.2-macos-aarch64-none/bin implementation = CPython uv = 0.7.6 version_info = 3.13.2 @@ -141,7 +141,7 @@ prompt = ruff extends-environment = /.other-environment ``` -`/doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin/python`: +`/do/re/mi//cpython-3.13.2-macos-aarch64-none/bin/python`: ```text ``` @@ -182,12 +182,12 @@ python = "/.venv" `/.venv/pyvenv.cfg`: ```cfg -home = /doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin +home = /do/re/mi//cpython-3.13.2-macos-aarch64-none/bin version_info = 3.13 command = /.pyenv/versions/3.13.3/bin/python3.13 -m venv --without-pip --prompt="python-default/3.13.3" /somewhere-else/python/virtualenvs/python-default/3.13.3 ``` -`/doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin/python`: +`/do/re/mi//cpython-3.13.2-macos-aarch64-none/bin/python`: ```text ``` diff --git a/crates/ty_python_semantic/resources/mdtest/import/workspaces.md b/crates/ty_python_semantic/resources/mdtest/import/workspaces.md index 4b6eb75165..95003765b8 100644 --- a/crates/ty_python_semantic/resources/mdtest/import/workspaces.md +++ b/crates/ty_python_semantic/resources/mdtest/import/workspaces.md @@ -240,10 +240,10 @@ python = "/.venv" `/.venv/pyvenv.cfg`: ```cfg -home = /doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin +home = /do/re/mi//cpython-3.13.2-macos-aarch64-none/bin ``` -`/doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin/python`: +`/do/re/mi//cpython-3.13.2-macos-aarch64-none/bin/python`: ```text ``` @@ -362,10 +362,10 @@ python = "/.venv" `/.venv/pyvenv.cfg`: ```cfg -home = /doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin +home = /do/re/mi//cpython-3.13.2-macos-aarch64-none/bin ``` -`/doo/doo/wop/cpython-3.13.2-macos-aarch64-none/bin/python`: +`/do/re/mi//cpython-3.13.2-macos-aarch64-none/bin/python`: ```text ``` From a9f2bb41bd802c9f4db417e3fec8324cf6207868 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Thu, 4 Dec 2025 08:12:04 +0100 Subject: [PATCH 10/41] [ty] Don't send publish diagnostics for clients supporting pull diagnostics (#21772) --- crates/ruff_db/src/system/path.rs | 7 ++ crates/ty_python_semantic/tests/corpus.rs | 3 +- .../notifications/did_change_watched_files.rs | 6 +- crates/ty_server/tests/e2e/code_actions.rs | 14 +-- crates/ty_server/tests/e2e/initialize.rs | 8 +- crates/ty_server/tests/e2e/inlay_hints.rs | 4 +- crates/ty_server/tests/e2e/main.rs | 12 +-- .../tests/e2e/publish_diagnostics.rs | 86 ++++++++++++++++++- .../ty_server/tests/e2e/pull_diagnostics.rs | 26 +++--- ...gnostics__on_did_change_watched_files.snap | 9 ++ 10 files changed, 138 insertions(+), 37 deletions(-) create mode 100644 crates/ty_server/tests/e2e/snapshots/e2e__publish_diagnostics__on_did_change_watched_files.snap diff --git a/crates/ruff_db/src/system/path.rs b/crates/ruff_db/src/system/path.rs index a387ae54f6..9c525f5aca 100644 --- a/crates/ruff_db/src/system/path.rs +++ b/crates/ruff_db/src/system/path.rs @@ -667,6 +667,13 @@ impl Deref for SystemPathBuf { } } +impl AsRef for SystemPathBuf { + #[inline] + fn as_ref(&self) -> &Path { + self.0.as_std_path() + } +} + impl> FromIterator

for SystemPathBuf { fn from_iter>(iter: I) -> Self { let mut buf = SystemPathBuf::new(); diff --git a/crates/ty_python_semantic/tests/corpus.rs b/crates/ty_python_semantic/tests/corpus.rs index c09929bcbd..505be61383 100644 --- a/crates/ty_python_semantic/tests/corpus.rs +++ b/crates/ty_python_semantic/tests/corpus.rs @@ -79,8 +79,7 @@ fn run_corpus_tests(pattern: &str) -> anyhow::Result<()> { let root = SystemPathBuf::from("/src"); let mut db = CorpusDb::new(); - db.memory_file_system() - .create_directory_all(root.as_ref())?; + db.memory_file_system().create_directory_all(&root)?; let workspace_root = get_cargo_workspace_root()?; let workspace_root = workspace_root.to_string(); diff --git a/crates/ty_server/src/server/api/notifications/did_change_watched_files.rs b/crates/ty_server/src/server/api/notifications/did_change_watched_files.rs index c67bbd8e70..fef69d9e66 100644 --- a/crates/ty_server/src/server/api/notifications/did_change_watched_files.rs +++ b/crates/ty_server/src/server/api/notifications/did_change_watched_files.rs @@ -1,6 +1,8 @@ use crate::document::DocumentKey; use crate::server::Result; -use crate::server::api::diagnostics::{publish_diagnostics, publish_settings_diagnostics}; +use crate::server::api::diagnostics::{ + publish_diagnostics_if_needed, publish_settings_diagnostics, +}; use crate::server::api::traits::{NotificationHandler, SyncNotificationHandler}; use crate::session::Session; use crate::session::client::Client; @@ -92,7 +94,7 @@ impl SyncNotificationHandler for DidChangeWatchedFiles { ); } else { for key in session.text_document_handles() { - publish_diagnostics(&key, session, client); + publish_diagnostics_if_needed(&key, session, client); } } diff --git a/crates/ty_server/tests/e2e/code_actions.rs b/crates/ty_server/tests/e2e/code_actions.rs index 77f4d42fcb..0e5eafcd13 100644 --- a/crates/ty_server/tests/e2e/code_actions.rs +++ b/crates/ty_server/tests/e2e/code_actions.rs @@ -65,7 +65,7 @@ unused-ignore-comment = \"warn\" .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); // Wait for diagnostics to be computed. let diagnostics = server.document_diagnostic_request(foo, None); @@ -103,7 +103,7 @@ unused-ignore-comment = \"warn\" .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); // Wait for diagnostics to be computed. let diagnostics = server.document_diagnostic_request(foo, None); @@ -145,7 +145,7 @@ unused-ignore-comment = \"warn\" .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); // Wait for diagnostics to be computed. let diagnostics = server.document_diagnostic_request(foo, None); @@ -182,7 +182,7 @@ def my_func(): ... .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); // Wait for diagnostics to be computed. let diagnostics = server.document_diagnostic_request(foo, None); @@ -221,7 +221,7 @@ def my_func(): ... .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); // Wait for diagnostics to be computed. let diagnostics = server.document_diagnostic_request(foo, None); @@ -257,7 +257,7 @@ x: typing.Literal[1] = 1 .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); // Wait for diagnostics to be computed. let diagnostics = server.document_diagnostic_request(foo, None); @@ -294,7 +294,7 @@ html.parser .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); // Wait for diagnostics to be computed. let diagnostics = server.document_diagnostic_request(foo, None); diff --git a/crates/ty_server/tests/e2e/initialize.rs b/crates/ty_server/tests/e2e/initialize.rs index 87910373b4..1526e78022 100644 --- a/crates/ty_server/tests/e2e/initialize.rs +++ b/crates/ty_server/tests/e2e/initialize.rs @@ -294,7 +294,7 @@ def foo() -> str: .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); let hover = server.hover_request(foo, Position::new(0, 5)); assert!( @@ -326,7 +326,7 @@ def foo() -> str: .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); let hover = server.hover_request(foo, Position::new(0, 5)); assert!( @@ -367,14 +367,14 @@ def bar() -> str: .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); let hover_foo = server.hover_request(foo, Position::new(0, 5)); assert!( hover_foo.is_none(), "Expected no hover information for workspace A, got: {hover_foo:?}" ); - server.open_text_document(bar, &bar_content, 1); + server.open_text_document(bar, bar_content, 1); let hover_bar = server.hover_request(bar, Position::new(0, 5)); assert!( hover_bar.is_some(), diff --git a/crates/ty_server/tests/e2e/inlay_hints.rs b/crates/ty_server/tests/e2e/inlay_hints.rs index 974f97c3de..2f2848d9b8 100644 --- a/crates/ty_server/tests/e2e/inlay_hints.rs +++ b/crates/ty_server/tests/e2e/inlay_hints.rs @@ -28,7 +28,7 @@ y = foo(1) .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); let _ = server.await_notification::(); let hints = server @@ -132,7 +132,7 @@ fn variable_inlay_hints_disabled() -> Result<()> { .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); let _ = server.await_notification::(); let hints = server diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 8e416d11a3..7e380562fa 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -742,15 +742,18 @@ impl TestServer { } pub(crate) fn file_uri(&self, path: impl AsRef) -> Url { - Url::from_file_path(self.test_context.root().join(path.as_ref()).as_std_path()) - .expect("Path must be a valid URL") + Url::from_file_path(self.file_path(path).as_std_path()).expect("Path must be a valid URL") + } + + pub(crate) fn file_path(&self, path: impl AsRef) -> SystemPathBuf { + self.test_context.root().join(path) } /// Send a `textDocument/didOpen` notification pub(crate) fn open_text_document( &mut self, path: impl AsRef, - content: &impl ToString, + content: impl AsRef, version: i32, ) { let params = DidOpenTextDocumentParams { @@ -758,7 +761,7 @@ impl TestServer { uri: self.file_uri(path), language_id: "python".to_string(), version, - text: content.to_string(), + text: content.as_ref().to_string(), }, }; self.send_notification::(params); @@ -793,7 +796,6 @@ impl TestServer { } /// Send a `workspace/didChangeWatchedFiles` notification with the given file events - #[expect(dead_code)] pub(crate) fn did_change_watched_files(&mut self, events: Vec) { let params = DidChangeWatchedFilesParams { changes: events }; self.send_notification::(params); diff --git a/crates/ty_server/tests/e2e/publish_diagnostics.rs b/crates/ty_server/tests/e2e/publish_diagnostics.rs index ae138506a6..b7f1eaf2d9 100644 --- a/crates/ty_server/tests/e2e/publish_diagnostics.rs +++ b/crates/ty_server/tests/e2e/publish_diagnostics.rs @@ -1,5 +1,7 @@ +use std::time::Duration; + use anyhow::Result; -use lsp_types::notification::PublishDiagnostics; +use lsp_types::{FileChangeType, FileEvent, notification::PublishDiagnostics}; use ruff_db::system::SystemPath; use crate::TestServerBuilder; @@ -20,10 +22,90 @@ def foo() -> str: .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); let diagnostics = server.await_notification::(); insta::assert_debug_snapshot!(diagnostics); Ok(()) } + +#[test] +fn on_did_change_watched_files() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.py"); + let foo_content = "\ +def foo() -> str: + print(a) +"; + + let mut server = TestServerBuilder::new()? + .with_workspace(workspace_root, None)? + .with_file(foo, "")? + .enable_pull_diagnostics(false) + .build() + .wait_until_workspaces_are_initialized(); + + let foo = server.file_path(foo); + + server.open_text_document(&foo, "", 1); + + let _open_diagnostics = server.await_notification::(); + + std::fs::write(&foo, foo_content)?; + + server.did_change_watched_files(vec![FileEvent { + uri: server.file_uri(foo), + typ: FileChangeType::CHANGED, + }]); + + let diagnostics = server.await_notification::(); + + // Note how ty reports no diagnostics here. This is because + // the contents received by didOpen/didChange take precedence over the file + // content on disk. Or, more specifically, because the revision + // of the file is not bumped, because it still uses the version + // from the `didOpen` notification but we don't have any notification + // that we can use here. + insta::assert_json_snapshot!(diagnostics); + + Ok(()) +} + +#[test] +fn on_did_change_watched_files_pull_diagnostics() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.py"); + let foo_content = "\ +def foo() -> str: + print(a) +"; + + let mut server = TestServerBuilder::new()? + .with_workspace(workspace_root, None)? + .with_file(foo, "")? + .enable_pull_diagnostics(true) + .build() + .wait_until_workspaces_are_initialized(); + + let foo = server.file_path(foo); + + server.open_text_document(&foo, "", 1); + + std::fs::write(&foo, foo_content)?; + + server.did_change_watched_files(vec![FileEvent { + uri: server.file_uri(foo), + typ: FileChangeType::CHANGED, + }]); + + let diagnostics = + server.try_await_notification::(Some(Duration::from_millis(100))); + + assert!( + diagnostics.is_err(), + "Server should not send a publish diagnostic notification if the client supports pull diagnostics" + ); + + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/pull_diagnostics.rs b/crates/ty_server/tests/e2e/pull_diagnostics.rs index ba67404d15..3256023905 100644 --- a/crates/ty_server/tests/e2e/pull_diagnostics.rs +++ b/crates/ty_server/tests/e2e/pull_diagnostics.rs @@ -31,7 +31,7 @@ def foo() -> str: .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); let diagnostics = server.document_diagnostic_request(foo, None); assert_debug_snapshot!(diagnostics); @@ -57,7 +57,7 @@ def foo() -> str: .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); // First request with no previous result ID let first_response = server.document_diagnostic_request(foo, None); @@ -113,7 +113,7 @@ def foo() -> str: .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content_v1, 1); + server.open_text_document(foo, foo_content_v1, 1); // First request with no previous result ID let first_response = server.document_diagnostic_request(foo, None); @@ -233,7 +233,7 @@ def foo() -> str: .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(file_a, &file_a_content, 1); + server.open_text_document(file_a, file_a_content, 1); // First request with no previous result IDs let mut first_response = server @@ -250,10 +250,10 @@ def foo() -> str: // Make changes to files B, C, D, and E (leave A unchanged) // Need to open files before changing them - server.open_text_document(file_b, &file_b_content_v1, 1); - server.open_text_document(file_c, &file_c_content_v1, 1); - server.open_text_document(file_d, &file_d_content_v1, 1); - server.open_text_document(file_e, &file_e_content_v1, 1); + server.open_text_document(file_b, file_b_content_v1, 1); + server.open_text_document(file_c, file_c_content_v1, 1); + server.open_text_document(file_d, file_d_content_v1, 1); + server.open_text_document(file_e, file_e_content_v1, 1); // File B: Add a new error server.change_text_document( @@ -536,9 +536,9 @@ fn workspace_diagnostic_streaming_with_caching() -> Result<()> { .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(SystemPath::new("src/error_0.py"), &error_content, 1); - server.open_text_document(SystemPath::new("src/error_1.py"), &error_content, 1); - server.open_text_document(SystemPath::new("src/error_2.py"), &error_content, 1); + server.open_text_document(SystemPath::new("src/error_0.py"), error_content, 1); + server.open_text_document(SystemPath::new("src/error_1.py"), error_content, 1); + server.open_text_document(SystemPath::new("src/error_2.py"), error_content, 1); // First request to get result IDs (non-streaming for simplicity) let first_response = server.workspace_diagnostic_request(None, None); @@ -716,7 +716,7 @@ def hello() -> str: create_workspace_server_with_file(workspace_root, file_path, file_content_no_error)?; // Open the file first - server.open_text_document(file_path, &file_content_no_error, 1); + server.open_text_document(file_path, file_content_no_error, 1); // Make a workspace diagnostic request to a project with one file but no diagnostics // This should trigger long-polling since the project has no diagnostics @@ -819,7 +819,7 @@ def hello() -> str: create_workspace_server_with_file(workspace_root, file_path, file_content_no_error)?; // Open the file first - server.open_text_document(file_path, &file_content_no_error, 1); + server.open_text_document(file_path, file_content_no_error, 1); // PHASE 1: Initial suspend (no diagnostics) let request_id_1 = send_workspace_diagnostic_request(&mut server); diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__publish_diagnostics__on_did_change_watched_files.snap b/crates/ty_server/tests/e2e/snapshots/e2e__publish_diagnostics__on_did_change_watched_files.snap new file mode 100644 index 0000000000..52ce909cad --- /dev/null +++ b/crates/ty_server/tests/e2e/snapshots/e2e__publish_diagnostics__on_did_change_watched_files.snap @@ -0,0 +1,9 @@ +--- +source: crates/ty_server/tests/e2e/publish_diagnostics.rs +expression: diagnostics +--- +{ + "uri": "file:///src/foo.py", + "diagnostics": [], + "version": 1 +} From 6491932757ca6365bd11b8241266d81c045f7a8b Mon Sep 17 00:00:00 2001 From: Aria Desires Date: Thu, 4 Dec 2025 03:11:40 -0500 Subject: [PATCH 11/41] [ty] Fix crash when hovering an unknown string annotation (#21782) ## Summary I have no idea what I'm doing with the fix (all the interesting stuff is in the second commit). The basic problem is the compiler emits the diagnostic: ``` x: "foobar" ^^^^^^ ``` Which the suppression code-action hands the end of to `Tokens::after` which then panics because that function panics if handed an offset that is in the middle of a token. Fixes https://github.com/astral-sh/ty/issues/1748 ## Test Plan Many tests added (only the e2e test matters). --- crates/ty_ide/src/find_references.rs | 36 +++++++++++++ crates/ty_ide/src/goto_declaration.rs | 35 ++++++++++++ crates/ty_ide/src/goto_type_definition.rs | 54 +++++++++++++++++++ crates/ty_ide/src/hover.rs | 54 +++++++++++++++++++ crates/ty_python_semantic/src/suppression.rs | 10 +++- crates/ty_server/tests/e2e/code_actions.rs | 36 +++++++++++++ ...ode_action_invalid_string_annotations.snap | 52 ++++++++++++++++++ 7 files changed, 275 insertions(+), 2 deletions(-) create mode 100644 crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_invalid_string_annotations.snap diff --git a/crates/ty_ide/src/find_references.rs b/crates/ty_ide/src/find_references.rs index d281dcaf92..5b5c3e4f43 100644 --- a/crates/ty_ide/src/find_references.rs +++ b/crates/ty_ide/src/find_references.rs @@ -898,6 +898,42 @@ cls = MyClass assert_snapshot!(test.references(), @"No references found"); } + #[test] + fn references_string_annotation_recursive() { + let test = cursor_test( + r#" + ab: "ab" + "#, + ); + + assert_snapshot!(test.references(), @r#" + info[references]: Reference 1 + --> main.py:2:1 + | + 2 | ab: "ab" + | ^^ + | + + info[references]: Reference 2 + --> main.py:2:6 + | + 2 | ab: "ab" + | ^^ + | + "#); + } + + #[test] + fn references_string_annotation_unknown() { + let test = cursor_test( + r#" + x: "foobar" + "#, + ); + + assert_snapshot!(test.references(), @"No references found"); + } + #[test] fn references_match_name_stmt() { let test = cursor_test( diff --git a/crates/ty_ide/src/goto_declaration.rs b/crates/ty_ide/src/goto_declaration.rs index 45efa4ae22..02b329f88e 100644 --- a/crates/ty_ide/src/goto_declaration.rs +++ b/crates/ty_ide/src/goto_declaration.rs @@ -1073,6 +1073,41 @@ def another_helper(path): assert_snapshot!(test.goto_declaration(), @"No goto target found"); } + #[test] + fn goto_declaration_string_annotation_recursive() { + let test = cursor_test( + r#" + ab: "ab" + "#, + ); + + assert_snapshot!(test.goto_declaration(), @r#" + info[goto-declaration]: Declaration + --> main.py:2:1 + | + 2 | ab: "ab" + | ^^ + | + info: Source + --> main.py:2:6 + | + 2 | ab: "ab" + | ^^ + | + "#); + } + + #[test] + fn goto_declaration_string_annotation_unknown() { + let test = cursor_test( + r#" + x: "foobar" + "#, + ); + + assert_snapshot!(test.goto_declaration(), @"No goto target found"); + } + #[test] fn goto_declaration_nested_instance_attribute() { let test = cursor_test( diff --git a/crates/ty_ide/src/goto_type_definition.rs b/crates/ty_ide/src/goto_type_definition.rs index 5964f241a4..b611eac28a 100644 --- a/crates/ty_ide/src/goto_type_definition.rs +++ b/crates/ty_ide/src/goto_type_definition.rs @@ -964,6 +964,60 @@ mod tests { assert_snapshot!(test.goto_type_definition(), @"No goto target found"); } + #[test] + fn goto_type_string_annotation_recursive() { + let test = cursor_test( + r#" + ab: "ab" + "#, + ); + + assert_snapshot!(test.goto_type_definition(), @r#" + info[goto-type-definition]: Type definition + --> stdlib/ty_extensions.pyi:20:1 + | + 19 | # Types + 20 | Unknown = object() + | ^^^^^^^ + 21 | AlwaysTruthy = object() + 22 | AlwaysFalsy = object() + | + info: Source + --> main.py:2:6 + | + 2 | ab: "ab" + | ^^ + | + "#); + } + + #[test] + fn goto_type_string_annotation_unknown() { + let test = cursor_test( + r#" + x: "foobar" + "#, + ); + + assert_snapshot!(test.goto_type_definition(), @r#" + info[goto-type-definition]: Type definition + --> stdlib/ty_extensions.pyi:20:1 + | + 19 | # Types + 20 | Unknown = object() + | ^^^^^^^ + 21 | AlwaysTruthy = object() + 22 | AlwaysFalsy = object() + | + info: Source + --> main.py:2:5 + | + 2 | x: "foobar" + | ^^^^^^ + | + "#); + } + #[test] fn goto_type_match_name_stmt() { let test = cursor_test( diff --git a/crates/ty_ide/src/hover.rs b/crates/ty_ide/src/hover.rs index 3b4b463ee2..e9d913570c 100644 --- a/crates/ty_ide/src/hover.rs +++ b/crates/ty_ide/src/hover.rs @@ -1089,6 +1089,60 @@ mod tests { assert_snapshot!(test.hover(), @"Hover provided no content"); } + #[test] + fn hover_string_annotation_recursive() { + let test = cursor_test( + r#" + ab: "ab" + "#, + ); + + assert_snapshot!(test.hover(), @r#" + Unknown + --------------------------------------------- + ```python + Unknown + ``` + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:2:6 + | + 2 | ab: "ab" + | ^- + | || + | |Cursor offset + | source + | + "#); + } + + #[test] + fn hover_string_annotation_unknown() { + let test = cursor_test( + r#" + x: "foobar" + "#, + ); + + assert_snapshot!(test.hover(), @r#" + Unknown + --------------------------------------------- + ```python + Unknown + ``` + --------------------------------------------- + info[hover]: Hovered content is + --> main.py:2:5 + | + 2 | x: "foobar" + | ^^^-^^ + | | | + | | Cursor offset + | source + | + "#); + } + #[test] fn hover_overload_type_disambiguated1() { let test = CursorTest::builder() diff --git a/crates/ty_python_semantic/src/suppression.rs b/crates/ty_python_semantic/src/suppression.rs index fd8df281e0..94590c076d 100644 --- a/crates/ty_python_semantic/src/suppression.rs +++ b/crates/ty_python_semantic/src/suppression.rs @@ -413,9 +413,15 @@ pub fn create_suppression_fix(db: &dyn Db, file: File, id: LintId, range: TextRa } // Always insert a new suppression at the end of the range to avoid having to deal with multiline strings - // etc. + // etc. Also make sure to not pass a sub-token range to `Tokens::after`. let parsed = parsed_module(db, file).load(db); - let tokens_after = parsed.tokens().after(range.end()); + let tokens = parsed.tokens().at_offset(range.end()); + let token_range = match tokens { + ruff_python_ast::token::TokenAt::None => range, + ruff_python_ast::token::TokenAt::Single(token) => token.range(), + ruff_python_ast::token::TokenAt::Between(..) => range, + }; + let tokens_after = parsed.tokens().after(token_range.end()); // Same as for `line_end` when building up the `suppressions`: Ignore newlines // in multiline-strings, inside f-strings, or after a line continuation because we can't diff --git a/crates/ty_server/tests/e2e/code_actions.rs b/crates/ty_server/tests/e2e/code_actions.rs index 0e5eafcd13..ba4e78f744 100644 --- a/crates/ty_server/tests/e2e/code_actions.rs +++ b/crates/ty_server/tests/e2e/code_actions.rs @@ -309,3 +309,39 @@ html.parser Ok(()) } + +/// Regression test for a panic when a code-fix diagnostic points at a string annotation +#[test] +fn code_action_invalid_string_annotations() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.py"); + let foo_content = r#" +ab: "foobar" +"#; + + let ty_toml = SystemPath::new("ty.toml"); + let ty_toml_content = ""; + + let mut server = TestServerBuilder::new()? + .with_workspace(workspace_root, None)? + .with_file(ty_toml, ty_toml_content)? + .with_file(foo, foo_content)? + .enable_pull_diagnostics(true) + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(foo, &foo_content, 1); + + // Wait for diagnostics to be computed. + let diagnostics = server.document_diagnostic_request(foo, None); + let range = full_range(foo_content); + let code_action_params = code_actions_at(&server, diagnostics, foo, range); + + // Get code actions for the line with the unused ignore comment. + let code_action_id = server.send_request::(code_action_params); + let code_actions = server.await_response::(&code_action_id); + + insta::assert_json_snapshot!(code_actions); + + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_invalid_string_annotations.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_invalid_string_annotations.snap new file mode 100644 index 0000000000..07ae5cb675 --- /dev/null +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_invalid_string_annotations.snap @@ -0,0 +1,52 @@ +--- +source: crates/ty_server/tests/e2e/code_actions.rs +expression: code_actions +--- +[ + { + "title": "Ignore 'unresolved-reference' for this line", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 1, + "character": 5 + }, + "end": { + "line": 1, + "character": 11 + } + }, + "severity": 1, + "code": "unresolved-reference", + "codeDescription": { + "href": "https://ty.dev/rules#unresolved-reference" + }, + "source": "ty", + "message": "Name `foobar` used when not defined", + "relatedInformation": [] + } + ], + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 1, + "character": 12 + }, + "end": { + "line": 1, + "character": 12 + } + }, + "newText": " # ty:ignore[unresolved-reference]" + } + ] + } + }, + "isPreferred": false + } +] From b8ecc83a54fd5d7955bf1ab4fb82fe18dcb52283 Mon Sep 17 00:00:00 2001 From: Dhruv Manilawala Date: Thu, 4 Dec 2025 16:20:37 +0530 Subject: [PATCH 12/41] Fix clippy errors on `main` (#21788) https://github.com/astral-sh/ruff/actions/runs/19922070773/job/57112827024#step:5:62 --- crates/ty_server/tests/e2e/code_actions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ty_server/tests/e2e/code_actions.rs b/crates/ty_server/tests/e2e/code_actions.rs index ba4e78f744..d60d9ad302 100644 --- a/crates/ty_server/tests/e2e/code_actions.rs +++ b/crates/ty_server/tests/e2e/code_actions.rs @@ -330,7 +330,7 @@ ab: "foobar" .build() .wait_until_workspaces_are_initialized(); - server.open_text_document(foo, &foo_content, 1); + server.open_text_document(foo, foo_content, 1); // Wait for diagnostics to be computed. let diagnostics = server.document_diagnostic_request(foo, None); From 3aefe85b32ff698b1a2086c2b50ff38af5c9dbed Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Thu, 4 Dec 2025 14:19:48 +0100 Subject: [PATCH 13/41] [ty] Ensure `rename` `CursorTest` calls `can_rename` before renaming (#21790) --- crates/ty_ide/src/rename.rs | 34 ++++++++-------------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/crates/ty_ide/src/rename.rs b/crates/ty_ide/src/rename.rs index 3ecc474d6d..62f1c9f1e4 100644 --- a/crates/ty_ide/src/rename.rs +++ b/crates/ty_ide/src/rename.rs @@ -110,6 +110,10 @@ mod tests { } fn rename(&self, new_name: &str) -> String { + let Some(_) = can_rename(&self.db, self.cursor.file, self.cursor.offset) else { + return "Cannot rename".to_string(); + }; + let Some(rename_results) = rename(&self.db, self.cursor.file, self.cursor.offset, new_name) else { @@ -1182,6 +1186,7 @@ result = func(10, y=20) "); } + // TODO Should rename the alias #[test] fn import_alias() { let test = CursorTest::builder() @@ -1197,21 +1202,10 @@ result = func(10, y=20) ) .build(); - assert_snapshot!(test.rename("z"), @r" - info[rename]: Rename symbol (found 2 locations) - --> main.py:3:20 - | - 2 | import warnings - 3 | import warnings as abc - | ^^^ - 4 | - 5 | x = abc - | --- - 6 | y = warnings - | - "); + assert_snapshot!(test.rename("z"), @"Cannot rename"); } + // TODO Should rename the alias #[test] fn import_alias_use() { let test = CursorTest::builder() @@ -1227,18 +1221,6 @@ result = func(10, y=20) ) .build(); - assert_snapshot!(test.rename("z"), @r" - info[rename]: Rename symbol (found 2 locations) - --> main.py:3:20 - | - 2 | import warnings - 3 | import warnings as abc - | ^^^ - 4 | - 5 | x = abc - | --- - 6 | y = warnings - | - "); + assert_snapshot!(test.rename("z"), @"Cannot rename"); } } From 326025d45f87548caba9a56c5606d80f85abc5ff Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Thu, 4 Dec 2025 14:40:16 +0100 Subject: [PATCH 14/41] [ty] Always register rename provider if client doesn't support dynamic registration (#21789) --- crates/ty_server/src/capabilities.rs | 15 +++++---------- crates/ty_server/src/server.rs | 11 ++--------- .../src/server/api/requests/prepare_rename.rs | 1 + crates/ty_server/src/session.rs | 2 +- .../e2e__initialize__initialization.snap | 3 +++ ...initialize__initialization_with_workspace.snap | 3 +++ 6 files changed, 15 insertions(+), 20 deletions(-) diff --git a/crates/ty_server/src/capabilities.rs b/crates/ty_server/src/capabilities.rs index 82837ec026..23daa43dee 100644 --- a/crates/ty_server/src/capabilities.rs +++ b/crates/ty_server/src/capabilities.rs @@ -1,5 +1,5 @@ use lsp_types::{ - ClientCapabilities, CodeActionKind, CodeActionOptions, CompletionOptions, + self as types, ClientCapabilities, CodeActionKind, CodeActionOptions, CompletionOptions, DeclarationCapability, DiagnosticOptions, DiagnosticServerCapabilities, HoverProviderCapability, InlayHintOptions, InlayHintServerCapabilities, MarkupKind, NotebookCellSelector, NotebookSelector, OneOf, RenameOptions, SelectionRangeProviderCapability, @@ -8,11 +8,9 @@ use lsp_types::{ TextDocumentSyncCapability, TextDocumentSyncKind, TextDocumentSyncOptions, TypeDefinitionProviderCapability, WorkDoneProgressOptions, }; +use std::str::FromStr; use crate::PositionEncoding; -use crate::session::GlobalSettings; -use lsp_types as types; -use std::str::FromStr; bitflags::bitflags! { /// Represents the resolved client capabilities for the language server. @@ -349,7 +347,6 @@ impl ResolvedClientCapabilities { pub(crate) fn server_capabilities( position_encoding: PositionEncoding, resolved_client_capabilities: ResolvedClientCapabilities, - global_settings: &GlobalSettings, ) -> ServerCapabilities { let diagnostic_provider = if resolved_client_capabilities.supports_diagnostic_dynamic_registration() { @@ -368,11 +365,9 @@ pub(crate) fn server_capabilities( // dynamically based on the `ty.experimental.rename` setting. None } else { - // Otherwise, we check whether user has enabled rename support via the resolved settings - // from initialization options. - global_settings - .is_rename_enabled() - .then(|| OneOf::Right(server_rename_options())) + // Otherwise, we always register the rename provider and bail out in `prepareRename` if + // the feature is disabled. + Some(OneOf::Right(server_rename_options())) }; ServerCapabilities { diff --git a/crates/ty_server/src/server.rs b/crates/ty_server/src/server.rs index 487febb4b2..321a74857e 100644 --- a/crates/ty_server/src/server.rs +++ b/crates/ty_server/src/server.rs @@ -72,15 +72,8 @@ impl Server { tracing::debug!("Resolved client capabilities: {resolved_client_capabilities}"); let position_encoding = Self::find_best_position_encoding(&client_capabilities); - let server_capabilities = server_capabilities( - position_encoding, - resolved_client_capabilities, - &initialization_options - .options - .global - .clone() - .into_settings(), - ); + let server_capabilities = + server_capabilities(position_encoding, resolved_client_capabilities); let version = ruff_db::program_version().unwrap_or("Unknown"); tracing::info!("Version: {version}"); diff --git a/crates/ty_server/src/server/api/requests/prepare_rename.rs b/crates/ty_server/src/server/api/requests/prepare_rename.rs index 2fd8228201..8601aa2995 100644 --- a/crates/ty_server/src/server/api/requests/prepare_rename.rs +++ b/crates/ty_server/src/server/api/requests/prepare_rename.rs @@ -32,6 +32,7 @@ impl BackgroundDocumentRequestHandler for PrepareRenameRequestHandler { if snapshot .workspace_settings() .is_language_services_disabled() + || !snapshot.global_settings().is_rename_enabled() { return Ok(None); } diff --git a/crates/ty_server/src/session.rs b/crates/ty_server/src/session.rs index 992f02f929..d97e11ac48 100644 --- a/crates/ty_server/src/session.rs +++ b/crates/ty_server/src/session.rs @@ -564,7 +564,7 @@ impl Session { publish_settings_diagnostics(self, client, root); } - if let Some(global_options) = combined_global_options.take() { + if let Some(global_options) = combined_global_options { let global_settings = global_options.into_settings(); if global_settings.diagnostic_mode().is_workspace() { for project in self.projects.values_mut() { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap index 79d71626b2..7a8cdce616 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap @@ -48,6 +48,9 @@ expression: initialization_result "quickfix" ] }, + "renameProvider": { + "prepareProvider": true + }, "declarationProvider": true, "executeCommandProvider": { "commands": [ diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap index 79d71626b2..7a8cdce616 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap @@ -48,6 +48,9 @@ expression: initialization_result "quickfix" ] }, + "renameProvider": { + "prepareProvider": true + }, "declarationProvider": true, "executeCommandProvider": { "commands": [ From 9d4f1c6ae24b75642a586531f4c668213fbac3fb Mon Sep 17 00:00:00 2001 From: Brent Westbrook <36778786+ntBre@users.noreply.github.com> Date: Thu, 4 Dec 2025 09:45:53 -0500 Subject: [PATCH 15/41] Bump 0.14.8 (#21791) --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ Cargo.lock | 6 +++--- README.md | 6 +++--- crates/ruff/Cargo.toml | 2 +- crates/ruff_linter/Cargo.toml | 2 +- crates/ruff_wasm/Cargo.toml | 2 +- docs/integrations.md | 8 ++++---- docs/tutorial.md | 2 +- pyproject.toml | 2 +- scripts/benchmarks/pyproject.toml | 2 +- 10 files changed, 45 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5f2dc1152..2508b4a54f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ # Changelog +## 0.14.8 + +Released on 2025-12-04. + +### Preview features + +- \[`flake8-bugbear`\] Catch `yield` expressions within other statements (`B901`) ([#21200](https://github.com/astral-sh/ruff/pull/21200)) +- \[`flake8-use-pathlib`\] Mark fixes unsafe for return type changes (`PTH104`, `PTH105`, `PTH109`, `PTH115`) ([#21440](https://github.com/astral-sh/ruff/pull/21440)) + +### Bug fixes + +- Fix syntax error false positives for `await` outside functions ([#21763](https://github.com/astral-sh/ruff/pull/21763)) +- \[`flake8-simplify`\] Fix truthiness assumption for non-iterable arguments in tuple/list/set calls (`SIM222`, `SIM223`) ([#21479](https://github.com/astral-sh/ruff/pull/21479)) + +### Documentation + +- Suggest using `--output-file` option in GitLab integration ([#21706](https://github.com/astral-sh/ruff/pull/21706)) + +### Other changes + +- [syntax-error] Default type parameter followed by non-default type parameter ([#21657](https://github.com/astral-sh/ruff/pull/21657)) + +### Contributors + +- [@kieran-ryan](https://github.com/kieran-ryan) +- [@11happy](https://github.com/11happy) +- [@danparizher](https://github.com/danparizher) +- [@ntBre](https://github.com/ntBre) + ## 0.14.7 Released on 2025-11-28. diff --git a/Cargo.lock b/Cargo.lock index 2dc5a258c8..6bc8bf881c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2859,7 +2859,7 @@ dependencies = [ [[package]] name = "ruff" -version = "0.14.7" +version = "0.14.8" dependencies = [ "anyhow", "argfile", @@ -3117,7 +3117,7 @@ dependencies = [ [[package]] name = "ruff_linter" -version = "0.14.7" +version = "0.14.8" dependencies = [ "aho-corasick", "anyhow", @@ -3473,7 +3473,7 @@ dependencies = [ [[package]] name = "ruff_wasm" -version = "0.14.7" +version = "0.14.8" dependencies = [ "console_error_panic_hook", "console_log", diff --git a/README.md b/README.md index a95fb77768..7e96c92479 100644 --- a/README.md +++ b/README.md @@ -147,8 +147,8 @@ curl -LsSf https://astral.sh/ruff/install.sh | sh powershell -c "irm https://astral.sh/ruff/install.ps1 | iex" # For a specific version. -curl -LsSf https://astral.sh/ruff/0.14.7/install.sh | sh -powershell -c "irm https://astral.sh/ruff/0.14.7/install.ps1 | iex" +curl -LsSf https://astral.sh/ruff/0.14.8/install.sh | sh +powershell -c "irm https://astral.sh/ruff/0.14.8/install.ps1 | iex" ``` You can also install Ruff via [Homebrew](https://formulae.brew.sh/formula/ruff), [Conda](https://anaconda.org/conda-forge/ruff), @@ -181,7 +181,7 @@ Ruff can also be used as a [pre-commit](https://pre-commit.com/) hook via [`ruff ```yaml - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.14.7 + rev: v0.14.8 hooks: # Run the linter. - id: ruff-check diff --git a/crates/ruff/Cargo.toml b/crates/ruff/Cargo.toml index a811287630..ff8516ebf2 100644 --- a/crates/ruff/Cargo.toml +++ b/crates/ruff/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ruff" -version = "0.14.7" +version = "0.14.8" publish = true authors = { workspace = true } edition = { workspace = true } diff --git a/crates/ruff_linter/Cargo.toml b/crates/ruff_linter/Cargo.toml index 786007b016..9d11a41e50 100644 --- a/crates/ruff_linter/Cargo.toml +++ b/crates/ruff_linter/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ruff_linter" -version = "0.14.7" +version = "0.14.8" publish = false authors = { workspace = true } edition = { workspace = true } diff --git a/crates/ruff_wasm/Cargo.toml b/crates/ruff_wasm/Cargo.toml index e688039563..a83171ab4b 100644 --- a/crates/ruff_wasm/Cargo.toml +++ b/crates/ruff_wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ruff_wasm" -version = "0.14.7" +version = "0.14.8" publish = false authors = { workspace = true } edition = { workspace = true } diff --git a/docs/integrations.md b/docs/integrations.md index 65553c6bdf..d92c785c76 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -80,7 +80,7 @@ You can add the following configuration to `.gitlab-ci.yml` to run a `ruff forma stage: build interruptible: true image: - name: ghcr.io/astral-sh/ruff:0.14.7-alpine + name: ghcr.io/astral-sh/ruff:0.14.8-alpine before_script: - cd $CI_PROJECT_DIR - ruff --version @@ -106,7 +106,7 @@ Ruff can be used as a [pre-commit](https://pre-commit.com) hook via [`ruff-pre-c ```yaml - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.14.7 + rev: v0.14.8 hooks: # Run the linter. - id: ruff-check @@ -119,7 +119,7 @@ To enable lint fixes, add the `--fix` argument to the lint hook: ```yaml - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.14.7 + rev: v0.14.8 hooks: # Run the linter. - id: ruff-check @@ -133,7 +133,7 @@ To avoid running on Jupyter Notebooks, remove `jupyter` from the list of allowed ```yaml - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.14.7 + rev: v0.14.8 hooks: # Run the linter. - id: ruff-check diff --git a/docs/tutorial.md b/docs/tutorial.md index 322b90ed5a..59b64e52e1 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -369,7 +369,7 @@ This tutorial has focused on Ruff's command-line interface, but Ruff can also be ```yaml - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.14.7 + rev: v0.14.8 hooks: # Run the linter. - id: ruff-check diff --git a/pyproject.toml b/pyproject.toml index 69ae365da6..5159682235 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "ruff" -version = "0.14.7" +version = "0.14.8" description = "An extremely fast Python linter and code formatter, written in Rust." authors = [{ name = "Astral Software Inc.", email = "hey@astral.sh" }] readme = "README.md" diff --git a/scripts/benchmarks/pyproject.toml b/scripts/benchmarks/pyproject.toml index 4f8b71cec6..e9bad66935 100644 --- a/scripts/benchmarks/pyproject.toml +++ b/scripts/benchmarks/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "scripts" -version = "0.14.7" +version = "0.14.8" description = "" authors = ["Charles Marsh "] From cccb0bbaa41e95a1efbe997ebe8119454f9b93f9 Mon Sep 17 00:00:00 2001 From: Aria Desires Date: Thu, 4 Dec 2025 10:46:23 -0500 Subject: [PATCH 16/41] [ty] Add tests for implicit submodule references (#21793) ## Summary I realized we don't really test `DefinitionKind::ImportFromSubmodule` in the IDE at all, so here's a bunch of them, just recording our current behaviour. ## Test Plan *stares at the camera* --- crates/ty_ide/src/find_references.rs | 207 +++++++++++++++ crates/ty_ide/src/goto_declaration.rs | 292 ++++++++++++++++++++++ crates/ty_ide/src/goto_type_definition.rs | 277 ++++++++++++++++++++ crates/ty_ide/src/hover.rs | 291 +++++++++++++++++++++ crates/ty_ide/src/rename.rs | 203 +++++++++++++++ 5 files changed, 1270 insertions(+) diff --git a/crates/ty_ide/src/find_references.rs b/crates/ty_ide/src/find_references.rs index 5b5c3e4f43..48cbfaf9cf 100644 --- a/crates/ty_ide/src/find_references.rs +++ b/crates/ty_ide/src/find_references.rs @@ -1906,4 +1906,211 @@ func_alias() | "); } + + #[test] + fn references_submodule_import_from_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = subpkg + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // TODO(submodule-imports): this should light up both instances of `subpkg` + assert_snapshot!(test.references(), @r" + info[references]: Reference 1 + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg.submod import val + 3 | + 4 | x = subpkg + | ^^^^^^ + | + "); + } + + #[test] + fn references_submodule_import_from_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = subpkg + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // TODO(submodule-imports): this should light up both instances of `subpkg` + assert_snapshot!(test.references(), @"No references found"); + } + + #[test] + fn references_submodule_import_from_wrong_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = submod + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // No references is actually correct (or it should only see itself) + assert_snapshot!(test.references(), @"No references found"); + } + + #[test] + fn references_submodule_import_from_wrong_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = submod + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // No references is actually correct (or it should only see itself) + assert_snapshot!(test.references(), @"No references found"); + } + + #[test] + fn references_submodule_import_from_confusing_shadowed_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // No references is actually correct (or it should only see itself) + assert_snapshot!(test.references(), @"No references found"); + } + + #[test] + fn references_submodule_import_from_confusing_real_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + assert_snapshot!(test.references(), @r" + info[references]: Reference 1 + --> mypackage/__init__.py:2:21 + | + 2 | from .subpkg import subpkg + | ^^^^^^ + 3 | + 4 | x = subpkg + | + + info[references]: Reference 2 + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg import subpkg + 3 | + 4 | x = subpkg + | ^^^^^^ + | + + info[references]: Reference 3 + --> mypackage/subpkg/__init__.py:2:1 + | + 2 | subpkg: int = 10 + | ^^^^^^ + | + "); + } + + #[test] + fn references_submodule_import_from_confusing_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // TODO: this should also highlight the RHS subpkg in the import + assert_snapshot!(test.references(), @r" + info[references]: Reference 1 + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg import subpkg + 3 | + 4 | x = subpkg + | ^^^^^^ + | + "); + } } diff --git a/crates/ty_ide/src/goto_declaration.rs b/crates/ty_ide/src/goto_declaration.rs index 02b329f88e..a2f147b2d3 100644 --- a/crates/ty_ide/src/goto_declaration.rs +++ b/crates/ty_ide/src/goto_declaration.rs @@ -2602,6 +2602,298 @@ def ab(a: int, *, c: int): ... "); } + #[test] + fn goto_declaration_submodule_import_from_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = subpkg + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // TODO(submodule-imports): this should only highlight `subpkg` in the import statement + // This happens because DefinitionKind::ImportFromSubmodule claims the entire ImportFrom node, + // which is correct but unhelpful. Unfortunately even if it only claimed the LHS identifier it + // would highlight `subpkg.submod` which is strictly better but still isn't what we want. + assert_snapshot!(test.goto_declaration(), @r" + info[goto-declaration]: Declaration + --> mypackage/__init__.py:2:1 + | + 2 | from .subpkg.submod import val + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + 3 | + 4 | x = subpkg + | + info: Source + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg.submod import val + 3 | + 4 | x = subpkg + | ^^^^^^ + | + "); + } + + #[test] + fn goto_declaration_submodule_import_from_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = subpkg + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // TODO(submodule-imports): I don't *think* this is what we want..? + // It's a bit confusing because this symbol is essentially the LHS *and* RHS of + // `subpkg = mypackage.subpkg`. As in, it's both defining a local `subpkg` and + // loading the module `mypackage.subpkg`, so, it's understandable to get confused! + assert_snapshot!(test.goto_declaration(), @r" + info[goto-declaration]: Declaration + --> mypackage/subpkg/__init__.py:1:1 + | + | + info: Source + --> mypackage/__init__.py:2:7 + | + 2 | from .subpkg.submod import val + | ^^^^^^ + 3 | + 4 | x = subpkg + | + "); + } + + #[test] + fn goto_declaration_submodule_import_from_wrong_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = submod + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // No result is correct! + assert_snapshot!(test.goto_declaration(), @"No goto target found"); + } + + #[test] + fn goto_declaration_submodule_import_from_wrong_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = submod + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // Going to the submod module is correct! + assert_snapshot!(test.goto_declaration(), @r" + info[goto-declaration]: Declaration + --> mypackage/subpkg/submod.py:1:1 + | + 1 | + | ^ + 2 | val: int = 0 + | + info: Source + --> mypackage/__init__.py:2:14 + | + 2 | from .subpkg.submod import val + | ^^^^^^ + 3 | + 4 | x = submod + | + "); + } + + #[test] + fn goto_declaration_submodule_import_from_confusing_shadowed_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // Going to the subpkg module is correct! + assert_snapshot!(test.goto_declaration(), @r" + info[goto-declaration]: Declaration + --> mypackage/subpkg/__init__.py:1:1 + | + 1 | + | ^ + 2 | subpkg: int = 10 + | + info: Source + --> mypackage/__init__.py:2:7 + | + 2 | from .subpkg import subpkg + | ^^^^^^ + 3 | + 4 | x = subpkg + | + "); + } + + #[test] + fn goto_declaration_submodule_import_from_confusing_real_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // Going to the subpkg `int` is correct! + assert_snapshot!(test.goto_declaration(), @r" + info[goto-declaration]: Declaration + --> mypackage/subpkg/__init__.py:2:1 + | + 2 | subpkg: int = 10 + | ^^^^^^ + | + info: Source + --> mypackage/__init__.py:2:21 + | + 2 | from .subpkg import subpkg + | ^^^^^^ + 3 | + 4 | x = subpkg + | + "); + } + + #[test] + fn goto_declaration_submodule_import_from_confusing_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // TODO(submodule-imports): Ok this one is FASCINATING and it's kinda right but confusing! + // + // So there's 3 relevant definitions here: + // + // * `subpkg: int = 10` in the other file is in fact the original definition + // + // * the LHS `subpkg` in the import is an instance of `subpkg = ...` + // because it's a `DefinitionKind::ImportFromSubmodle`. + // This is the span that covers the entire import. + // + // * `the RHS `subpkg` in the import is a second instance of `subpkg = ...` + // that *immediately* overwrites the `ImportFromSubmodule`'s definition + // This span seemingly doesn't appear at all!? Is it getting hidden by the LHS span? + assert_snapshot!(test.goto_declaration(), @r" + info[goto-declaration]: Declaration + --> mypackage/__init__.py:2:1 + | + 2 | from .subpkg import subpkg + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ + 3 | + 4 | x = subpkg + | + info: Source + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg import subpkg + 3 | + 4 | x = subpkg + | ^^^^^^ + | + + info[goto-declaration]: Declaration + --> mypackage/subpkg/__init__.py:2:1 + | + 2 | subpkg: int = 10 + | ^^^^^^ + | + info: Source + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg import subpkg + 3 | + 4 | x = subpkg + | ^^^^^^ + | + "); + } + impl CursorTest { fn goto_declaration(&self) -> String { let Some(targets) = goto_declaration(&self.db, self.cursor.file, self.cursor.offset) diff --git a/crates/ty_ide/src/goto_type_definition.rs b/crates/ty_ide/src/goto_type_definition.rs index b611eac28a..fc5aa9aded 100644 --- a/crates/ty_ide/src/goto_type_definition.rs +++ b/crates/ty_ide/src/goto_type_definition.rs @@ -1672,6 +1672,283 @@ def function(): "#); } + #[test] + fn goto_type_submodule_import_from_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = subpkg + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // The module is the correct type definition + assert_snapshot!(test.goto_type_definition(), @r" + info[goto-type-definition]: Type definition + --> mypackage/subpkg/__init__.py:1:1 + | + | + info: Source + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg.submod import val + 3 | + 4 | x = subpkg + | ^^^^^^ + | + "); + } + + #[test] + fn goto_type_submodule_import_from_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = subpkg + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // The module is the correct type definition + assert_snapshot!(test.goto_type_definition(), @r" + info[goto-type-definition]: Type definition + --> mypackage/subpkg/__init__.py:1:1 + | + | + info: Source + --> mypackage/__init__.py:2:7 + | + 2 | from .subpkg.submod import val + | ^^^^^^ + 3 | + 4 | x = subpkg + | + "); + } + + #[test] + fn goto_type_submodule_import_from_wrong_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = submod + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // Unknown is correct, `submod` is not in scope + assert_snapshot!(test.goto_type_definition(), @r" + info[goto-type-definition]: Type definition + --> stdlib/ty_extensions.pyi:20:1 + | + 19 | # Types + 20 | Unknown = object() + | ^^^^^^^ + 21 | AlwaysTruthy = object() + 22 | AlwaysFalsy = object() + | + info: Source + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg.submod import val + 3 | + 4 | x = submod + | ^^^^^^ + | + "); + } + + #[test] + fn goto_type_submodule_import_from_wrong_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = submod + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // The module is correct + assert_snapshot!(test.goto_type_definition(), @r" + info[goto-type-definition]: Type definition + --> mypackage/subpkg/submod.py:1:1 + | + 1 | / + 2 | | val: int = 0 + | |_____________^ + | + info: Source + --> mypackage/__init__.py:2:14 + | + 2 | from .subpkg.submod import val + | ^^^^^^ + 3 | + 4 | x = submod + | + "); + } + + #[test] + fn goto_type_submodule_import_from_confusing_shadowed_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // The module is correct + assert_snapshot!(test.goto_type_definition(), @r" + info[goto-type-definition]: Type definition + --> mypackage/subpkg/__init__.py:1:1 + | + 1 | / + 2 | | subpkg: int = 10 + | |_________________^ + | + info: Source + --> mypackage/__init__.py:2:7 + | + 2 | from .subpkg import subpkg + | ^^^^^^ + 3 | + 4 | x = subpkg + | + "); + } + + #[test] + fn goto_type_submodule_import_from_confusing_real_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // `int` is correct + assert_snapshot!(test.goto_type_definition(), @r#" + info[goto-type-definition]: Type definition + --> stdlib/builtins.pyi:348:7 + | + 347 | @disjoint_base + 348 | class int: + | ^^^ + 349 | """int([x]) -> integer + 350 | int(x, base=10) -> integer + | + info: Source + --> mypackage/__init__.py:2:21 + | + 2 | from .subpkg import subpkg + | ^^^^^^ + 3 | + 4 | x = subpkg + | + "#); + } + + #[test] + fn goto_type_submodule_import_from_confusing_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // `int` is correct + assert_snapshot!(test.goto_type_definition(), @r#" + info[goto-type-definition]: Type definition + --> stdlib/builtins.pyi:348:7 + | + 347 | @disjoint_base + 348 | class int: + | ^^^ + 349 | """int([x]) -> integer + 350 | int(x, base=10) -> integer + | + info: Source + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg import subpkg + 3 | + 4 | x = subpkg + | ^^^^^^ + | + "#); + } + impl CursorTest { fn goto_type_definition(&self) -> String { let Some(targets) = diff --git a/crates/ty_ide/src/hover.rs b/crates/ty_ide/src/hover.rs index e9d913570c..affa6054e2 100644 --- a/crates/ty_ide/src/hover.rs +++ b/crates/ty_ide/src/hover.rs @@ -3321,6 +3321,297 @@ def function(): "); } + #[test] + fn hover_submodule_import_from_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = subpkg + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // The module is correct + assert_snapshot!(test.hover(), @r" + + --------------------------------------------- + ```python + + ``` + --------------------------------------------- + info[hover]: Hovered content is + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg.submod import val + 3 | + 4 | x = subpkg + | ^^^-^^ + | | | + | | Cursor offset + | source + | + "); + } + + #[test] + fn hover_submodule_import_from_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = subpkg + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // The module is correct + assert_snapshot!(test.hover(), @r" + + --------------------------------------------- + ```python + + ``` + --------------------------------------------- + info[hover]: Hovered content is + --> mypackage/__init__.py:2:7 + | + 2 | from .subpkg.submod import val + | ^^^-^^ + | | | + | | Cursor offset + | source + 3 | + 4 | x = subpkg + | + "); + } + + #[test] + fn hover_submodule_import_from_wrong_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = submod + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // Unknown is correct + assert_snapshot!(test.hover(), @r" + Unknown + --------------------------------------------- + ```python + Unknown + ``` + --------------------------------------------- + info[hover]: Hovered content is + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg.submod import val + 3 | + 4 | x = submod + | ^^^-^^ + | | | + | | Cursor offset + | source + | + "); + } + + #[test] + fn hover_submodule_import_from_wrong_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = submod + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // The submodule is correct + assert_snapshot!(test.hover(), @r" + + --------------------------------------------- + ```python + + ``` + --------------------------------------------- + info[hover]: Hovered content is + --> mypackage/__init__.py:2:14 + | + 2 | from .subpkg.submod import val + | ^^^-^^ + | | | + | | Cursor offset + | source + 3 | + 4 | x = submod + | + "); + } + + #[test] + fn hover_submodule_import_from_confusing_shadowed_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // The module is correct + assert_snapshot!(test.hover(), @r" + + --------------------------------------------- + ```python + + ``` + --------------------------------------------- + info[hover]: Hovered content is + --> mypackage/__init__.py:2:7 + | + 2 | from .subpkg import subpkg + | ^^^-^^ + | | | + | | Cursor offset + | source + 3 | + 4 | x = subpkg + | + "); + } + + #[test] + fn hover_submodule_import_from_confusing_real_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // int is correct + assert_snapshot!(test.hover(), @r" + int + --------------------------------------------- + ```python + int + ``` + --------------------------------------------- + info[hover]: Hovered content is + --> mypackage/__init__.py:2:21 + | + 2 | from .subpkg import subpkg + | ^^^-^^ + | | | + | | Cursor offset + | source + 3 | + 4 | x = subpkg + | + "); + } + + #[test] + fn hover_submodule_import_from_confusing_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // int is correct + assert_snapshot!(test.hover(), @r" + int + --------------------------------------------- + ```python + int + ``` + --------------------------------------------- + info[hover]: Hovered content is + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg import subpkg + 3 | + 4 | x = subpkg + | ^^^-^^ + | | | + | | Cursor offset + | source + | + "); + } + impl CursorTest { fn hover(&self) -> String { use std::fmt::Write; diff --git a/crates/ty_ide/src/rename.rs b/crates/ty_ide/src/rename.rs index 62f1c9f1e4..10ef9c197f 100644 --- a/crates/ty_ide/src/rename.rs +++ b/crates/ty_ide/src/rename.rs @@ -1223,4 +1223,207 @@ result = func(10, y=20) assert_snapshot!(test.rename("z"), @"Cannot rename"); } + + #[test] + fn rename_submodule_import_from_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = subpkg + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // TODO(submodule-imports): we should refuse to rename this (it's the name of a module) + assert_snapshot!(test.rename("mypkg"), @r" + info[rename]: Rename symbol (found 1 locations) + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg.submod import val + 3 | + 4 | x = subpkg + | ^^^^^^ + | + "); + } + + #[test] + fn rename_submodule_import_from_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = subpkg + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // Refusing to rename is correct + assert_snapshot!(test.rename("mypkg"), @"Cannot rename"); + } + + #[test] + fn rename_submodule_import_from_wrong_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = submod + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // Refusing to rename is good/fine here, it's an undefined reference + assert_snapshot!(test.rename("mypkg"), @"Cannot rename"); + } + + #[test] + fn rename_submodule_import_from_wrong_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg.submod import val + + x = submod + "#, + ) + .source("mypackage/subpkg/__init__.py", r#""#) + .source( + "mypackage/subpkg/submod.py", + r#" + val: int = 0 + "#, + ) + .build(); + + // Refusing to rename is good here, it's a module name + assert_snapshot!(test.rename("mypkg"), @"Cannot rename"); + } + + #[test] + fn rename_submodule_import_from_confusing_shadowed_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // Refusing to rename is good here, it's the name of a module + assert_snapshot!(test.rename("mypkg"), @"Cannot rename"); + } + + #[test] + fn rename_submodule_import_from_confusing_real_def() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // Renaming the integer is correct + assert_snapshot!(test.rename("mypkg"), @r" + info[rename]: Rename symbol (found 3 locations) + --> mypackage/__init__.py:2:21 + | + 2 | from .subpkg import subpkg + | ^^^^^^ + 3 | + 4 | x = subpkg + | ------ + | + ::: mypackage/subpkg/__init__.py:2:1 + | + 2 | subpkg: int = 10 + | ------ + | + "); + } + + #[test] + fn rename_submodule_import_from_confusing_use() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from .subpkg import subpkg + + x = subpkg + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + subpkg: int = 10 + "#, + ) + .build(); + + // TODO(submodule-imports): this is incorrect, we should rename the `subpkg` int + // and the RHS of the import statement (but *not* rename the LHS). + // + // However us being cautious here *would* be good as the rename will actually + // result in a `subpkg` variable still existing in this code, as the import's LHS + // `DefinitionKind::ImportFromSubmodule` would stop being overwritten by the RHS! + assert_snapshot!(test.rename("mypkg"), @r" + info[rename]: Rename symbol (found 1 locations) + --> mypackage/__init__.py:4:5 + | + 2 | from .subpkg import subpkg + 3 | + 4 | x = subpkg + | ^^^^^^ + | + "); + } } From 62f20b1e86494331551fa7ae8d6eb584658e11a7 Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Wed, 29 Oct 2025 15:20:48 -0400 Subject: [PATCH 17/41] [ty] Re-arrange imports in symbol extraction I like using a qualified `ast::` prefix for things from `ruff_python_ast`, so switch over to that convention. --- crates/ty_ide/src/symbols.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/ty_ide/src/symbols.rs b/crates/ty_ide/src/symbols.rs index a2278f40e6..ebb6637ae3 100644 --- a/crates/ty_ide/src/symbols.rs +++ b/crates/ty_ide/src/symbols.rs @@ -9,8 +9,9 @@ use regex::Regex; use ruff_db::files::File; use ruff_db::parsed::parsed_module; use ruff_index::{IndexVec, newtype_index}; +use ruff_python_ast as ast; +use ruff_python_ast::name::Name; use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor}; -use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange}; use ty_project::Db; @@ -416,7 +417,7 @@ struct SymbolVisitor { } impl SymbolVisitor { - fn visit_body(&mut self, body: &[Stmt]) { + fn visit_body(&mut self, body: &[ast::Stmt]) { for stmt in body { self.visit_stmt(stmt); } @@ -457,9 +458,9 @@ impl SymbolVisitor { } impl SourceOrderVisitor<'_> for SymbolVisitor { - fn visit_stmt(&mut self, stmt: &Stmt) { + fn visit_stmt(&mut self, stmt: &ast::Stmt) { match stmt { - Stmt::FunctionDef(func_def) => { + ast::Stmt::FunctionDef(func_def) => { let kind = if self .iter_symbol_stack() .any(|s| s.kind == SymbolKind::Class) @@ -501,7 +502,7 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { self.pop_symbol(); } - Stmt::ClassDef(class_def) => { + ast::Stmt::ClassDef(class_def) => { let symbol = SymbolTree { parent: None, name: class_def.name.to_string(), @@ -521,13 +522,15 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { self.pop_symbol(); } - Stmt::Assign(assign) => { + ast::Stmt::Assign(assign) => { // Include assignments only when we're in global or class scope if self.in_function { return; } for target in &assign.targets { - let Expr::Name(name) = target else { continue }; + let ast::Expr::Name(name) = target else { + continue; + }; let kind = if Self::is_constant_name(name.id.as_str()) { SymbolKind::Constant } else if self @@ -550,12 +553,12 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { } } - Stmt::AnnAssign(ann_assign) => { + ast::Stmt::AnnAssign(ann_assign) => { // Include assignments only when we're in global or class scope if self.in_function { return; } - let Expr::Name(name) = &*ann_assign.target else { + let ast::Expr::Name(name) = &*ann_assign.target else { return; }; let kind = if Self::is_constant_name(name.id.as_str()) { From 5da45f8ec78f119ae58cd04b8634b6831a423efa Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Tue, 25 Nov 2025 14:02:34 -0500 Subject: [PATCH 18/41] [ty] Simplify auto-import AST visitor slightly and add tests This simplifies the existing visitor by DRYing it up slightly. We also add tests for the existing functionality. In particular, we want to add support for re-export conventions, and that warrants more careful testing. --- crates/ty_ide/src/symbols.rs | 266 +++++++++++++++++++++++++++++------ 1 file changed, 221 insertions(+), 45 deletions(-) diff --git a/crates/ty_ide/src/symbols.rs b/crates/ty_ide/src/symbols.rs index ebb6637ae3..a49ac318fc 100644 --- a/crates/ty_ide/src/symbols.rs +++ b/crates/ty_ide/src/symbols.rs @@ -436,6 +436,28 @@ impl SymbolVisitor { symbol_id } + fn add_assignment(&mut self, stmt: &ast::Stmt, name: &ast::ExprName) -> SymbolId { + let kind = if Self::is_constant_name(name.id.as_str()) { + SymbolKind::Constant + } else if self + .iter_symbol_stack() + .any(|s| s.kind == SymbolKind::Class) + { + SymbolKind::Field + } else { + SymbolKind::Variable + }; + + let symbol = SymbolTree { + parent: None, + name: name.id.to_string(), + kind, + name_range: name.range(), + full_range: stmt.range(), + }; + self.add_symbol(symbol) + } + fn push_symbol(&mut self, symbol: SymbolTree) { let symbol_id = self.add_symbol(symbol); self.symbol_stack.push(symbol_id); @@ -501,7 +523,6 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { self.pop_symbol(); } - ast::Stmt::ClassDef(class_def) => { let symbol = SymbolTree { parent: None, @@ -521,7 +542,6 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { source_order::walk_stmt(self, stmt); self.pop_symbol(); } - ast::Stmt::Assign(assign) => { // Include assignments only when we're in global or class scope if self.in_function { @@ -531,28 +551,9 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { let ast::Expr::Name(name) = target else { continue; }; - let kind = if Self::is_constant_name(name.id.as_str()) { - SymbolKind::Constant - } else if self - .iter_symbol_stack() - .any(|s| s.kind == SymbolKind::Class) - { - SymbolKind::Field - } else { - SymbolKind::Variable - }; - - let symbol = SymbolTree { - parent: None, - name: name.id.to_string(), - kind, - name_range: name.range(), - full_range: stmt.range(), - }; - self.add_symbol(symbol); + self.add_assignment(stmt, name); } } - ast::Stmt::AnnAssign(ann_assign) => { // Include assignments only when we're in global or class scope if self.in_function { @@ -561,27 +562,8 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { let ast::Expr::Name(name) = &*ann_assign.target else { return; }; - let kind = if Self::is_constant_name(name.id.as_str()) { - SymbolKind::Constant - } else if self - .iter_symbol_stack() - .any(|s| s.kind == SymbolKind::Class) - { - SymbolKind::Field - } else { - SymbolKind::Variable - }; - - let symbol = SymbolTree { - parent: None, - name: name.id.to_string(), - kind, - name_range: name.range(), - full_range: stmt.range(), - }; - self.add_symbol(symbol); + self.add_assignment(stmt, name); } - _ => { source_order::walk_stmt(self, stmt); } @@ -591,9 +573,16 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { #[cfg(test)] mod tests { - fn matches(query: &str, symbol: &str) -> bool { - super::QueryPattern::fuzzy(query).is_match_symbol_name(symbol) - } + use camino::Utf8Component; + use insta::internals::SettingsBindDropGuard; + + use ruff_db::Db; + use ruff_db::files::{FileRootKind, system_path_to_file}; + use ruff_db::system::{DbWithWritableSystem, SystemPath, SystemPathBuf}; + use ruff_python_trivia::textwrap::dedent; + use ty_project::{ProjectMetadata, TestDb}; + + use super::symbols_for_file_global_only; #[test] fn various_yes() { @@ -625,4 +614,191 @@ mod tests { assert!(!matches("abcd", "abc")); assert!(!matches("δΘπ", "θΔΠ")); } + + #[test] + fn exports_simple() { + insta::assert_snapshot!( + public_test("\ +FOO = 1 +foo = 1 +frob: int = 1 +class Foo: + BAR = 1 +def quux(): + baz = 1 +").exports(), + @r" + FOO :: Constant + foo :: Variable + frob :: Variable + Foo :: Class + quux :: Function + ", + ); + } + + #[test] + fn exports_conditional_true() { + insta::assert_snapshot!( + public_test("\ +foo = 1 +if True: + bar = 1 +").exports(), + @r" + foo :: Variable + bar :: Variable + ", + ); + } + + #[test] + fn exports_conditional_false() { + // FIXME: This shouldn't include `bar`. + insta::assert_snapshot!( + public_test("\ +foo = 1 +if False: + bar = 1 +").exports(), + @r" + foo :: Variable + bar :: Variable + ", + ); + } + + #[test] + fn exports_conditional_sys_version() { + // FIXME: This shouldn't include `bar`. + insta::assert_snapshot!( + public_test("\ +import sys + +foo = 1 +if sys.version < (3, 5): + bar = 1 +").exports(), + @r" + foo :: Variable + bar :: Variable + ", + ); + } + + #[test] + fn exports_type_checking() { + insta::assert_snapshot!( + public_test("\ +from typing import TYPE_CHECKING + +foo = 1 +if TYPE_CHECKING: + bar = 1 +").exports(), + @r" + foo :: Variable + bar :: Variable + ", + ); + } + + fn matches(query: &str, symbol: &str) -> bool { + super::QueryPattern::fuzzy(query).is_match_symbol_name(symbol) + } + + fn public_test(code: &str) -> PublicTest { + PublicTestBuilder::default().source("test.py", code).build() + } + + struct PublicTest { + db: TestDb, + _insta_settings_guard: SettingsBindDropGuard, + } + + impl PublicTest { + /// Returns the exports from `test.py`. + /// + /// This is, conventionally, the default module file path used. For + /// example, it's used by the `public_test` convenience constructor. + fn exports(&self) -> String { + self.exports_for("test.py") + } + + /// Returns the exports from the module at the given path. + /// + /// The path given must have been written to this test's salsa DB. + fn exports_for(&self, path: impl AsRef) -> String { + let file = system_path_to_file(&self.db, path.as_ref()).unwrap(); + let symbols = symbols_for_file_global_only(&self.db, file); + symbols + .iter() + .map(|(_, symbol)| { + format!("{name} :: {kind:?}", name = symbol.name, kind = symbol.kind) + }) + .collect::>() + .join("\n") + } + } + + #[derive(Default)] + struct PublicTestBuilder { + /// A list of source files, corresponding to the + /// file's path and its contents. + sources: Vec, + } + + impl PublicTestBuilder { + pub(super) fn build(&self) -> PublicTest { + let mut db = TestDb::new(ProjectMetadata::new( + "test".into(), + SystemPathBuf::from("/"), + )); + + db.init_program().unwrap(); + + for Source { path, contents } in &self.sources { + db.write_file(path, contents) + .expect("write to memory file system to be successful"); + + // Add a root for the top-most component. + let top = path.components().find_map(|c| match c { + Utf8Component::Normal(c) => Some(c), + _ => None, + }); + if let Some(top) = top { + let top = SystemPath::new(top); + if db.system().is_directory(top) { + db.files() + .try_add_root(&db, top, FileRootKind::LibrarySearchPath); + } + } + } + + // N.B. We don't set anything custom yet, but we leave + // this here for when we invevitable add a filter. + let insta_settings = insta::Settings::clone_current(); + let insta_settings_guard = insta_settings.bind_to_scope(); + PublicTest { + db, + _insta_settings_guard: insta_settings_guard, + } + } + + pub(super) fn source( + &mut self, + path: impl Into, + contents: impl AsRef, + ) -> &mut PublicTestBuilder { + let path = path.into(); + let contents = dedent(contents.as_ref()).into_owned(); + self.sources.push(Source { path, contents }); + self + } + } + + struct Source { + path: SystemPathBuf, + contents: String, + } } From 086f1e0b8992b9b0231e7503484eba5bbd379dbb Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Thu, 4 Dec 2025 12:09:15 -0500 Subject: [PATCH 19/41] [ty] Skip over expressions in auto-import AST scanning --- crates/ty_ide/src/symbols.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/ty_ide/src/symbols.rs b/crates/ty_ide/src/symbols.rs index a49ac318fc..295ccad5c3 100644 --- a/crates/ty_ide/src/symbols.rs +++ b/crates/ty_ide/src/symbols.rs @@ -569,6 +569,10 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { } } } + + // TODO: We might consider handling walrus expressions + // here, since they can be used to introduce new names. + fn visit_expr(&mut self, _expr: &ast::Expr) {} } #[cfg(test)] From 8c72b296c9895b9e40dacf69c584a62b84f09803 Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Tue, 25 Nov 2025 14:03:19 -0500 Subject: [PATCH 20/41] [ty] Add support for re-exports and `__all__` to auto-import This commit (mostly) re-implements the support for `__all__` in ty-proper, but inside the auto-import AST scanner. When `__all__` isn't present in a module, we fall back to conventions to determine whether a symbol is exported or not: https://docs.python.org/3/library/index.html However, in keeping with current practice for non-auto-import completions, we continue to provide sunder and dunder names as re-exports. When `__all__` is present, we respect it strictly. That is, a symbol is exported *if and only if* it's in `__all__`. This is somewhat stricter than pylance seemingly is. I felt like it was a good idea to start here, and we can relax it based on user demand (perhaps through a setting). --- crates/ty_ide/src/symbols.rs | 1308 ++++++++++++++++- .../snapshots/e2e__notebook__auto_import.snap | 4 +- .../e2e__notebook__auto_import_docstring.snap | 4 +- ...2e__notebook__auto_import_from_future.snap | 4 +- .../e2e__notebook__auto_import_same_cell.snap | 4 +- 5 files changed, 1293 insertions(+), 31 deletions(-) diff --git a/crates/ty_ide/src/symbols.rs b/crates/ty_ide/src/symbols.rs index 295ccad5c3..a80a9ed56d 100644 --- a/crates/ty_ide/src/symbols.rs +++ b/crates/ty_ide/src/symbols.rs @@ -13,7 +13,9 @@ use ruff_python_ast as ast; use ruff_python_ast::name::Name; use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor}; use ruff_text_size::{Ranged, TextRange}; +use rustc_hash::FxHashSet; use ty_project::Db; +use ty_python_semantic::{ModuleName, resolve_module}; use crate::completion::CompletionKind; @@ -111,7 +113,13 @@ impl PartialEq for QueryPattern { /// A flat list of indexed symbols for a single file. #[derive(Clone, Debug, Default, PartialEq, Eq, get_size2::GetSize)] pub struct FlatSymbols { + /// The symbols exported by a module. symbols: IndexVec, + /// The names found in an `__all__` for a module. + /// + /// This is `None` if the module has no `__all__` at module + /// scope. + all_names: Option>, } impl FlatSymbols { @@ -351,16 +359,9 @@ pub(crate) fn symbols_for_file(db: &dyn Db, file: File) -> FlatSymbols { let parsed = parsed_module(db, file); let module = parsed.load(db); - let mut visitor = SymbolVisitor { - symbols: IndexVec::new(), - symbol_stack: vec![], - in_function: false, - global_only: false, - }; + let mut visitor = SymbolVisitor::tree(db, file); visitor.visit_body(&module.syntax().body); - FlatSymbols { - symbols: visitor.symbols, - } + visitor.into_flat_symbols() } /// Returns a flat list of *only global* symbols in the file given. @@ -373,12 +374,7 @@ pub(crate) fn symbols_for_file_global_only(db: &dyn Db, file: File) -> FlatSymbo let parsed = parsed_module(db, file); let module = parsed.load(db); - let mut visitor = SymbolVisitor { - symbols: IndexVec::new(), - symbol_stack: vec![], - in_function: false, - global_only: true, - }; + let mut visitor = SymbolVisitor::globals(db, file); visitor.visit_body(&module.syntax().body); if file @@ -389,10 +385,7 @@ pub(crate) fn symbols_for_file_global_only(db: &dyn Db, file: File) -> FlatSymbo // Eagerly clear ASTs of third party files. parsed.clear(); } - - FlatSymbols { - symbols: visitor.symbols, - } + visitor.into_flat_symbols() } #[derive(Debug, Clone, PartialEq, Eq, get_size2::GetSize)] @@ -402,27 +395,122 @@ struct SymbolTree { kind: SymbolKind, name_range: TextRange, full_range: TextRange, + import_kind: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, get_size2::GetSize)] +enum ImportKind { + Normal, + RedundantAlias, + Wildcard, } /// A visitor over all symbols in a single file. /// /// This guarantees that child symbols have a symbol ID greater /// than all of its parents. -struct SymbolVisitor { +#[allow(clippy::struct_excessive_bools)] +struct SymbolVisitor<'db> { + db: &'db dyn Db, + file: File, symbols: IndexVec, symbol_stack: Vec, - /// Track if we're currently inside a function (to exclude local variables) + /// Track if we're currently inside a function at any point. + /// + /// This is true even when we're inside a class definition + /// that is inside a class. in_function: bool, + /// Track if we're currently inside a class at any point. + /// + /// This is true even when we're inside a function definition + /// that is inside a class. + in_class: bool, global_only: bool, + /// The origin of an `__all__` variable, if found. + all_origin: Option, + /// A set of names extracted from `__all__`. + all_names: FxHashSet, + /// A flag indicating whether the module uses unrecognized + /// `__all__` idioms or there are any invalid elements in + /// `__all__`. + all_invalid: bool, } -impl SymbolVisitor { +impl<'db> SymbolVisitor<'db> { + fn tree(db: &'db dyn Db, file: File) -> Self { + Self { + db, + file, + symbols: IndexVec::new(), + symbol_stack: vec![], + in_function: false, + in_class: false, + global_only: false, + all_origin: None, + all_names: FxHashSet::default(), + all_invalid: false, + } + } + + fn globals(db: &'db dyn Db, file: File) -> Self { + Self { + global_only: true, + ..Self::tree(db, file) + } + } + + fn into_flat_symbols(mut self) -> FlatSymbols { + // We want to filter out some of the symbols we collected. + // Specifically, to respect conventions around library + // interface. + // + // But, we always assigned IDs to each symbol based on + // their position in a sequence. So when we filter some + // out, we need to remap the identifiers. + // + // N.B. The remapping could be skipped when `global_only` is + // true, since in that case, none of the symbols have a parent + // ID by construction. + let mut remap = IndexVec::with_capacity(self.symbols.len()); + let mut new = IndexVec::with_capacity(self.symbols.len()); + for mut symbol in std::mem::take(&mut self.symbols) { + if !self.is_part_of_library_interface(&symbol) { + remap.push(None); + continue; + } + + if let Some(ref mut parent) = symbol.parent { + // OK because the visitor guarantees that + // all parents have IDs less than their + // children. So its ID has already been + // remapped. + if let Some(new_parent) = remap[*parent] { + *parent = new_parent; + } else { + // The parent symbol was dropped, so + // all of its children should be as + // well. + remap.push(None); + continue; + } + } + let new_id = new.next_index(); + remap.push(Some(new_id)); + new.push(symbol); + } + FlatSymbols { + symbols: new, + all_names: self.all_origin.map(|_| self.all_names), + } + } + fn visit_body(&mut self, body: &[ast::Stmt]) { for stmt in body { self.visit_stmt(stmt); } } + /// Add a new symbol and return its ID. fn add_symbol(&mut self, mut symbol: SymbolTree) -> SymbolId { if let Some(&parent_id) = self.symbol_stack.last() { symbol.parent = Some(parent_id); @@ -436,6 +524,7 @@ impl SymbolVisitor { symbol_id } + /// Adds a symbol introduced via an assignment. fn add_assignment(&mut self, stmt: &ast::Stmt, name: &ast::ExprName) -> SymbolId { let kind = if Self::is_constant_name(name.id.as_str()) { SymbolKind::Constant @@ -454,10 +543,222 @@ impl SymbolVisitor { kind, name_range: name.range(), full_range: stmt.range(), + import_kind: None, }; self.add_symbol(symbol) } + /// Adds a symbol introduced via an import `stmt`. + fn add_import_alias(&mut self, stmt: &ast::Stmt, alias: &ast::Alias) -> SymbolId { + let name = alias.asname.as_ref().unwrap_or(&alias.name); + let kind = if stmt.is_import_stmt() { + SymbolKind::Module + } else if Self::is_constant_name(name.as_str()) { + SymbolKind::Constant + } else { + SymbolKind::Variable + }; + let re_export = Some( + if alias.asname.as_ref().map(ast::Identifier::as_str) == Some(alias.name.as_str()) { + ImportKind::RedundantAlias + } else { + ImportKind::Normal + }, + ); + self.add_symbol(SymbolTree { + parent: None, + name: name.id.to_string(), + kind, + name_range: name.range(), + full_range: stmt.range(), + import_kind: re_export, + }) + } + + /// Extracts `__all__` names from the given assignment. + /// + /// If the assignment isn't for `__all__`, then this is a no-op. + fn add_all_assignment(&mut self, targets: &[ast::Expr], value: Option<&ast::Expr>) { + if self.in_function || self.in_class { + return; + } + let Some(target) = targets.first() else { + return; + }; + if !is_dunder_all(target) { + return; + } + + let Some(value) = value else { return }; + match *value { + // `__all__ = [...]` + // `__all__ = (...)` + ast::Expr::List(ast::ExprList { ref elts, .. }) + | ast::Expr::Tuple(ast::ExprTuple { ref elts, .. }) => { + self.update_all_origin(DunderAllOrigin::CurrentModule); + if !self.add_all_names(elts) { + self.all_invalid = true; + } + } + _ => { + self.all_invalid = true; + } + } + } + + /// Extends the current set of names with the names from the + /// given expression which currently must be a list/tuple/set of + /// string-literal names. This currently does not support using a + /// submodule's `__all__` variable. + /// + /// Returns `true` if the expression is a valid list/tuple/set or + /// module `__all__`, `false` otherwise. + /// + /// N.B. Supporting all instances of `__all__ += submodule.__all__` + /// and `__all__.extend(submodule.__all__)` is likely difficult + /// in this context. Namely, `submodule` needs to be resolved + /// to a particular module. ty proper can do this (by virtue + /// of inferring the type of `submodule`). With that said, we + /// could likely support a subset of cases here without too much + /// ceremony. ---AG + fn extend_all(&mut self, expr: &ast::Expr) -> bool { + match expr { + // `__all__ += [...]` + // `__all__ += (...)` + // `__all__ += {...}` + ast::Expr::List(ast::ExprList { elts, .. }) + | ast::Expr::Tuple(ast::ExprTuple { elts, .. }) + | ast::Expr::Set(ast::ExprSet { elts, .. }) => self.add_all_names(elts), + _ => false, + } + } + + /// Processes a call idiom for `__all__` and updates the set of + /// names accordingly. + /// + /// Returns `true` if the call idiom is recognized and valid, + /// `false` otherwise. + fn update_all_by_call_idiom( + &mut self, + function_name: &ast::Identifier, + arguments: &ast::Arguments, + ) -> bool { + if arguments.len() != 1 { + return false; + } + let Some(argument) = arguments.find_positional(0) else { + return false; + }; + match function_name.as_str() { + // `__all__.extend([...])` + // `__all__.extend(module.__all__)` + "extend" => { + if !self.extend_all(argument) { + return false; + } + } + // `__all__.append(...)` + "append" => { + let Some(name) = create_all_name(argument) else { + return false; + }; + self.all_names.insert(name); + } + // `__all__.remove(...)` + "remove" => { + let Some(name) = create_all_name(argument) else { + return false; + }; + self.all_names.remove(&name); + } + _ => return false, + } + true + } + + /// Adds all of the names exported from the module + /// imported by `import_from`. i.e., This implements + /// `from module import *` semantics. + fn add_exported_from_wildcard(&mut self, import_from: &ast::StmtImportFrom) { + let Some(symbols) = self.get_names_from_wildcard(import_from) else { + self.all_invalid = true; + return; + }; + self.symbols + .extend(symbols.symbols.iter().filter_map(|symbol| { + // If there's no `__all__`, then names with an underscore + // are never pulled in via a wildcard import. Otherwise, + // we defer to `__all__` filtering. + if symbols.all_names.is_none() && symbol.name.starts_with('_') { + return None; + } + let mut symbol = symbol.clone(); + symbol.import_kind = Some(ImportKind::Wildcard); + Some(symbol) + })); + // If the imported module defines an `__all__` AND `__all__` is + // in `__all__`, then the importer gets it too. + if let Some(ref all) = symbols.all_names + && all.contains("__all__") + { + self.update_all_origin(DunderAllOrigin::StarImport); + self.all_names.extend(all.iter().cloned()); + } + } + + /// Adds `__all__` from the module imported by `import_from`. i.e., + /// This implements `from module import __all__` semantics. + fn add_all_from_import(&mut self, import_from: &ast::StmtImportFrom) { + let Some(symbols) = self.get_names_from_wildcard(import_from) else { + self.all_invalid = true; + return; + }; + // If the imported module defines an `__all__`, + // then the importer gets it too. + if let Some(ref all) = symbols.all_names { + self.update_all_origin(DunderAllOrigin::ExternalModule); + self.all_names.extend(all.iter().cloned()); + } + } + + /// Returns the exported symbols (along with `__all__`) from the + /// module imported in `import_from`. + fn get_names_from_wildcard( + &self, + import_from: &ast::StmtImportFrom, + ) -> Option<&'db FlatSymbols> { + let module_name = + ModuleName::from_import_statement(self.db, self.file, import_from).ok()?; + let module = resolve_module(self.db, self.file, &module_name)?; + Some(symbols_for_file_global_only(self.db, module.file(self.db)?)) + } + + /// Add valid names from `__all__` to the set of existing `__all__` + /// names. + /// + /// Returns `false` if any of the names are invalid. + fn add_all_names(&mut self, exprs: &[ast::Expr]) -> bool { + for expr in exprs { + let Some(name) = create_all_name(expr) else { + return false; + }; + self.all_names.insert(name); + } + true + } + + /// Updates the origin of `__all__` in the current module. + /// + /// This will clear existing names if the origin is changed to + /// mimic the behavior of overriding `__all__` in the current + /// module. + fn update_all_origin(&mut self, origin: DunderAllOrigin) { + if self.all_origin.is_some() { + self.all_names.clear(); + } + self.all_origin = Some(origin); + } + fn push_symbol(&mut self, symbol: SymbolTree) { let symbol_id = self.add_symbol(symbol); self.symbol_stack.push(symbol_id); @@ -477,9 +778,62 @@ impl SymbolVisitor { fn is_constant_name(name: &str) -> bool { name.chars().all(|c| c.is_ascii_uppercase() || c == '_') } + + /// This routine determines whether the given symbol should be + /// considered part of the public API of this module. The given + /// symbol should defined or imported into this module. + /// + /// See: + fn is_part_of_library_interface(&self, symbol: &SymbolTree) -> bool { + // If this is a child of something else, then we always + // defer its visibility to the parent. + if symbol.parent.is_some() { + return true; + } + + // When there's no `__all__`, we use conventions to determine + // if a name should be part of the exported API of a module + // or not. When there is `__all__`, we currently follow it + // strictly. + if self.all_origin.is_some() { + // If `__all__` is somehow invalid, ignore it and fall + // through as-if `__all__` didn't exist. + if self.all_invalid { + tracing::debug!("Invalid `__all__` in `{}`", self.file.path(self.db)); + } else { + return self.all_names.contains(&*symbol.name); + } + } + + // "Imported symbols are considered private by default. A fixed + // set of import forms re-export imported symbols." Specifically: + // + // * `import X as X` + // * `from Y import X as X` + // * `from Y import *` + if let Some(kind) = symbol.import_kind { + return match kind { + ImportKind::RedundantAlias | ImportKind::Wildcard => true, + ImportKind::Normal => false, + }; + } + // "Symbols whose names begin with an underscore (but are not + // dunder names) are considered private." + // + // ... however, we currently include these as part of the public + // API. The only extant (2025-12-03) consumer is completions, and + // completions will rank these names lower than others. + if symbol.name.starts_with('_') + && !(symbol.name.starts_with("__") && symbol.name.ends_with("__")) + { + return true; + } + // ... otherwise, it's exported! + true + } } -impl SourceOrderVisitor<'_> for SymbolVisitor { +impl SourceOrderVisitor<'_> for SymbolVisitor<'_> { fn visit_stmt(&mut self, stmt: &ast::Stmt) { match stmt { ast::Stmt::FunctionDef(func_def) => { @@ -502,6 +856,7 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { kind, name_range: func_def.name.range(), full_range: stmt.range(), + import_kind: None, }; if self.global_only { @@ -530,6 +885,7 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { kind: SymbolKind::Class, name_range: class_def.name.range(), full_range: stmt.range(), + import_kind: None, }; if self.global_only { @@ -538,11 +894,20 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { return; } + // Mark that we're entering a class scope + let was_in_class = self.in_class; + self.in_class = true; + self.push_symbol(symbol); source_order::walk_stmt(self, stmt); self.pop_symbol(); + + // Restore the previous class scope state + self.in_class = was_in_class; } ast::Stmt::Assign(assign) => { + self.add_all_assignment(&assign.targets, Some(&assign.value)); + // Include assignments only when we're in global or class scope if self.in_function { return; @@ -555,6 +920,11 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { } } ast::Stmt::AnnAssign(ann_assign) => { + self.add_all_assignment( + std::slice::from_ref(&ann_assign.target), + ann_assign.value.as_deref(), + ); + // Include assignments only when we're in global or class scope if self.in_function { return; @@ -564,6 +934,89 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { }; self.add_assignment(stmt, name); } + ast::Stmt::AugAssign(ast::StmtAugAssign { + target, op, value, .. + }) => { + if self.all_origin.is_none() { + // We can't update `__all__` if it doesn't already + // exist. + return; + } + if !is_dunder_all(target) { + return; + } + // Anything other than `+=` is not valid. + if !matches!(op, ast::Operator::Add) { + self.all_invalid = true; + return; + } + if !self.extend_all(value) { + self.all_invalid = true; + } + } + ast::Stmt::Expr(expr) => { + if self.all_origin.is_none() { + // We can't update `__all__` if it doesn't already exist. + return; + } + let Some(ast::ExprCall { + func, arguments, .. + }) = expr.value.as_call_expr() + else { + return; + }; + let Some(ast::ExprAttribute { + value, + attr, + ctx: ast::ExprContext::Load, + .. + }) = func.as_attribute_expr() + else { + return; + }; + if !is_dunder_all(value) { + return; + } + if !self.update_all_by_call_idiom(attr, arguments) { + self.all_invalid = true; + } + + source_order::walk_stmt(self, stmt); + } + ast::Stmt::Import(import) => { + // We only consider imports in global scope. + if self.in_function { + return; + } + for alias in &import.names { + self.add_import_alias(stmt, alias); + } + } + ast::Stmt::ImportFrom(import_from) => { + // We only consider imports in global scope. + if self.in_function { + return; + } + for alias in &import_from.names { + if &alias.name == "*" { + self.add_exported_from_wildcard(import_from); + } else { + if &alias.name == "__all__" + && alias + .asname + .as_ref() + .is_none_or(|asname| asname == "__all__") + { + self.add_all_from_import(import_from); + } + self.add_import_alias(stmt, alias); + } + } + } + // FIXME: We don't currently try to evaluate `if` + // statements. We just assume that all `if` statements are + // always `True`. This applies to symbols in general but + // also `__all__`. _ => { source_order::walk_stmt(self, stmt); } @@ -575,6 +1028,29 @@ impl SourceOrderVisitor<'_> for SymbolVisitor { fn visit_expr(&mut self, _expr: &ast::Expr) {} } +/// Represents where an `__all__` has been defined. +#[derive(Debug, Clone)] +enum DunderAllOrigin { + /// The `__all__` variable is defined in the current module. + CurrentModule, + /// The `__all__` variable is imported from another module. + ExternalModule, + /// The `__all__` variable is imported from a module via a `*`-import. + StarImport, +} + +/// Checks if the given expression is a name expression for `__all__`. +fn is_dunder_all(expr: &ast::Expr) -> bool { + matches!(expr, ast::Expr::Name(ast::ExprName { id, .. }) if id == "__all__") +} + +/// Create and return a string representing a name from the given +/// expression, or `None` if it is an invalid expression for a +/// `__all__` element. +fn create_all_name(expr: &ast::Expr) -> Option { + Some(expr.as_string_literal_expr()?.value.to_str().into()) +} + #[cfg(test)] mod tests { use camino::Utf8Component; @@ -641,6 +1117,25 @@ def quux(): ); } + /// The typing spec says that names beginning with an underscore + /// ought to be considered unexported[1]. However, at present, we + /// currently include them in completions but rank them lower than + /// non-underscore names. So this tests that we return underscore + /// names. + /// + /// [1]: https://typing.python.org/en/latest/spec/distributing.html#library-interface-public-and-private-symbols + #[test] + fn exports_underscore() { + insta::assert_snapshot!( + public_test("\ +_foo = 1 +").exports(), + @r" + _foo :: Variable + ", + ); + } + #[test] fn exports_conditional_true() { insta::assert_snapshot!( @@ -707,6 +1202,773 @@ if TYPE_CHECKING: ); } + #[test] + fn exports_conditional_always_else() { + // FIXME: This shouldn't include `bar`. + insta::assert_snapshot!( + public_test("\ +foo = 1 +bar = 1 +if True: + __all__ = ['foo'] +else: + __all__ = ['foo', 'bar'] +").exports(), + @r" + foo :: Variable + bar :: Variable + ", + ); + } + + #[test] + fn exports_all_overwrites_previous() { + insta::assert_snapshot!( + public_test("\ +foo = 1 +bar = 1 +__all__ = ['foo'] +__all__ = ['foo', 'bar'] +").exports(), + @r" + foo :: Variable + bar :: Variable + ", + ); + } + + #[test] + fn exports_import_no_reexport() { + insta::assert_snapshot!( + public_test("\ +import collections +").exports(), + @r"", + ); + } + + #[test] + fn exports_import_as_no_reexport() { + insta::assert_snapshot!( + public_test("\ +import numpy as np +").exports(), + @r"", + ); + } + + #[test] + fn exports_from_import_no_reexport() { + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +").exports(), + @r"", + ); + } + + #[test] + fn exports_from_import_as_no_reexport() { + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict as dd +").exports(), + @r"", + ); + } + + #[test] + fn exports_import_reexport() { + insta::assert_snapshot!( + public_test("\ +import numpy as numpy +").exports(), + @"numpy :: Module", + ); + } + + #[test] + fn exports_from_import_reexport() { + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict as defaultdict +").exports(), + @"defaultdict :: Variable", + ); + } + + #[test] + fn exports_from_import_all_reexport_assignment() { + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__ = ['defaultdict'] +").exports(), + @"defaultdict :: Variable", + ); + + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__ = ('defaultdict',) +").exports(), + @"defaultdict :: Variable", + ); + } + + #[test] + fn exports_from_import_all_reexport_annotated_assignment() { + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__: list[str] = ['defaultdict'] +").exports(), + @"defaultdict :: Variable", + ); + + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__: tuple[str, ...] = ('defaultdict',) +").exports(), + @"defaultdict :: Variable", + ); + } + + #[test] + fn exports_from_import_all_reexport_augmented_assignment() { + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__ = [] +__all__ += ['defaultdict'] +").exports(), + @"defaultdict :: Variable", + ); + + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__ = [] +__all__ += ('defaultdict',) +").exports(), + @"defaultdict :: Variable", + ); + + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__ = [] +__all__ += {'defaultdict'} +").exports(), + @"defaultdict :: Variable", + ); + } + + #[test] + fn exports_from_import_all_reexport_invalid_augmented_assignment() { + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__ += ['defaultdict'] +").exports(), + @"", + ); + } + + #[test] + fn exports_from_import_all_reexport_extend() { + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__ = [] +__all__.extend(['defaultdict']) +").exports(), + @"defaultdict :: Variable", + ); + } + + #[test] + fn exports_from_import_all_reexport_invalid_extend() { + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__.extend(['defaultdict']) +").exports(), + @r"", + ); + } + + #[test] + fn exports_from_import_all_reexport_append() { + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__ = [] +__all__.append('defaultdict') +").exports(), + @"defaultdict :: Variable", + ); + } + + #[test] + fn exports_from_import_all_reexport_plus_equals() { + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__ = [] +__all__ += ['defaultdict'] +").exports(), + @"defaultdict :: Variable", + ); + } + + #[test] + fn exports_from_import_all_reexport_star_equals() { + // Confirm that this doesn't work. Only `__all__ += ...` should + // be recognized. This makes the symbol visitor consider + // `__all__` invalid and thus ignore it. And this in turn lets + // `__all__` be exported. This seems like a somewhat degenerate + // case, but is a consequence of us treating sunder and dunder + // symbols as exported when `__all__` isn't present. + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__ = [] +__all__ *= ['defaultdict'] +").exports(), + @"__all__ :: Variable", + ); + } + + #[test] + fn exports_from_import_all_reexport_remove() { + insta::assert_snapshot!( + public_test("\ +from collections import defaultdict +__all__ = [] +__all__.remove('defaultdict') +").exports(), + @"", + ); + } + + #[test] + fn exports_nested_all() { + insta::assert_snapshot!( + public_test(r#"\ +bar = 1 +baz = 1 +__all__ = [] + +def foo(): + __all__.append("bar") + +class X: + def method(self): + __all__.extend(["baz"]) +"#).exports(), + @"", + ); + } + + #[test] + fn wildcard_reexport_simple_no_all() { + let test = PublicTestBuilder::default() + .source("foo.py", "ZQZQZQ = 1") + .source("test.py", "from foo import *") + .build(); + insta::assert_snapshot!( + test.exports_for("test.py"), + @"ZQZQZQ :: Constant", + ); + } + + #[test] + fn wildcard_reexport_single_underscore_no_all() { + let test = PublicTestBuilder::default() + .source("foo.py", "_ZQZQZQ = 1") + .source("test.py", "from foo import *") + .build(); + // Without `__all__` present, a wildcard import won't include + // names starting with an underscore at runtime. So `_ZQZQZQ` + // should not be present here. + // See: + insta::assert_snapshot!( + test.exports_for("test.py"), + @"", + ); + } + + #[test] + fn wildcard_reexport_double_underscore_no_all() { + let test = PublicTestBuilder::default() + .source("foo.py", "__ZQZQZQ = 1") + .source("test.py", "from foo import *") + .build(); + // Without `__all__` present, a wildcard import won't include + // names starting with an underscore at runtime. So `__ZQZQZQ` + // should not be present here. + // See: + insta::assert_snapshot!( + test.exports_for("test.py"), + @"", + ); + } + + #[test] + fn wildcard_reexport_normal_import_no_all() { + let test = PublicTestBuilder::default() + .source("foo.py", "import collections") + .source("test.py", "from foo import *") + .build(); + // We specifically test for the absence of `collections` + // here. That is, `from foo import *` will import + // `collections` at runtime, but we don't consider it part + // of the exported interface of `foo`. + insta::assert_snapshot!( + test.exports_for("test.py"), + @"", + ); + } + + #[test] + fn wildcard_reexport_redundant_import_no_all() { + let test = PublicTestBuilder::default() + .source("foo.py", "import collections as collections") + .source("test.py", "from foo import *") + .build(); + insta::assert_snapshot!( + test.exports_for("test.py"), + @"collections :: Module", + ); + } + + #[test] + fn wildcard_reexport_normal_from_import_no_all() { + let test = PublicTestBuilder::default() + .source("foo.py", "from collections import defaultdict") + .source("test.py", "from foo import *") + .build(); + // We specifically test for the absence of `defaultdict` + // here. That is, `from foo import *` will import + // `defaultdict` at runtime, but we don't consider it part + // of the exported interface of `foo`. + insta::assert_snapshot!( + test.exports_for("test.py"), + @"", + ); + } + + #[test] + fn wildcard_reexport_redundant_from_import_no_all() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + "from collections import defaultdict as defaultdict", + ) + .source("test.py", "from foo import *") + .build(); + insta::assert_snapshot!( + test.exports_for("test.py"), + @"defaultdict :: Variable", + ); + } + + #[test] + fn wildcard_reexport_all_simple() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = ['ZQZQZQ'] + ", + ) + .source("test.py", "from foo import *") + .build(); + insta::assert_snapshot!( + test.exports_for("test.py"), + @"ZQZQZQ :: Constant", + ); + } + + #[test] + fn wildcard_reexport_all_simple_include_all() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = ['__all__', 'ZQZQZQ'] + ", + ) + .source("test.py", "from foo import *") + .build(); + insta::assert_snapshot!( + test.exports_for("test.py"), + @r" + ZQZQZQ :: Constant + __all__ :: Variable + ", + ); + } + + #[test] + fn wildcard_reexport_all_empty() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = [] + ", + ) + .source("test.py", "from foo import *") + .build(); + // Nothing is exported because `__all__` is defined + // and also empty. + insta::assert_snapshot!( + test.exports_for("test.py"), + @"", + ); + } + + #[test] + fn wildcard_reexport_all_empty_not_applies_to_importer() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = [] + ", + ) + .source( + "test.py", + "from foo import * + TRICKSY = 1", + ) + .build(); + insta::assert_snapshot!( + test.exports_for("test.py"), + @"TRICKSY :: Constant", + ); + } + + #[test] + fn wildcard_reexport_all_include_all_applies_to_importer() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = ['__all__'] + ", + ) + .source( + "test.py", + "from foo import * + TRICKSY = 1", + ) + .build(); + // TRICKSY should specifically be absent because + // `__all__` is defined in `test.py` (via a wildcard + // import) and does not itself include `TRICKSY`. + insta::assert_snapshot!( + test.exports_for("test.py"), + @"__all__ :: Variable", + ); + } + + #[test] + fn wildcard_reexport_all_empty_then_added_to() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = [] + ", + ) + .source( + "test.py", + "from foo import * + TRICKSY = 1 + __all__.append('TRICKSY')", + ) + .build(); + insta::assert_snapshot!( + test.exports_for("test.py"), + @"TRICKSY :: Constant", + ); + } + + #[test] + fn wildcard_reexport_all_include_all_then_added_to() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = ['__all__'] + ", + ) + .source( + "test.py", + "from foo import * + TRICKSY = 1 + __all__.append('TRICKSY')", + ) + .build(); + insta::assert_snapshot!( + test.exports_for("test.py"), + @r" + __all__ :: Variable + TRICKSY :: Constant + ", + ); + } + + /// Tests that a `from module import *` doesn't bring an + /// `__all__` into scope if `module` doesn't provide an + /// `__all__` that includes `__all__` AND this causes + /// `__all__.append` to fail in the importing module + /// (because it isn't defined). + #[test] + fn wildcard_reexport_all_empty_then_added_to_incorrect() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = [] + ", + ) + .source( + "test.py", + "from foo import * + from collections import defaultdict + __all__.append('defaultdict')", + ) + .build(); + insta::assert_snapshot!( + test.exports_for("test.py"), + @"", + ); + } + + #[test] + fn wildcard_reexport_all_include_all_then_added_to_correct() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = ['__all__'] + ", + ) + .source( + "test.py", + "from foo import * + from collections import defaultdict + __all__.append('defaultdict')", + ) + .build(); + insta::assert_snapshot!( + test.exports_for("test.py"), + @r" + __all__ :: Variable + defaultdict :: Variable + ", + ); + } + + #[test] + fn wildcard_reexport_all_non_empty_but_non_existent() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = ['TRICKSY'] + ", + ) + .source("test.py", "from foo import *") + .build(); + // `TRICKSY` isn't actually a valid symbol, + // and `ZQZQZQ` isn't in `__all__`, so we get + // no symbols here. + insta::assert_snapshot!( + test.exports_for("test.py"), + @"", + ); + } + + #[test] + fn wildcard_reexport_all_include_all_and_non_existent() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = ['__all__', 'TRICKSY'] + ", + ) + .source("test.py", "from foo import *") + .build(); + // Note that this example will actually result in a runtime + // error since `TRICKSY` doesn't exist in `foo.py` and + // `from foo import *` will try to import it anyway. + insta::assert_snapshot!( + test.exports_for("test.py"), + @"__all__ :: Variable", + ); + } + + #[test] + fn wildcard_reexport_all_not_applies_to_importer() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = ['TRICKSY'] + ", + ) + .source( + "test.py", + "from foo import * + TRICKSY = 1", + ) + .build(); + // Note that this example will actually result in a runtime + // error since `TRICKSY` doesn't exist in `foo.py` and + // `from foo import *` will try to import it anyway. + insta::assert_snapshot!( + test.exports_for("test.py"), + @"TRICKSY :: Constant", + ); + } + + #[test] + fn wildcard_reexport_all_include_all_with_others_applies_to_importer() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = ['__all__', 'TRICKSY'] + ", + ) + .source( + "test.py", + "from foo import * + TRICKSY = 1", + ) + .build(); + // Note that this example will actually result in a runtime + // error since `TRICKSY` doesn't exist in `foo.py` and + // `from foo import *` will try to import it anyway. + insta::assert_snapshot!( + test.exports_for("test.py"), + @r" + __all__ :: Variable + TRICKSY :: Constant + ", + ); + } + + #[test] + fn explicit_reexport_all_empty_applies_to_importer() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = [] + ", + ) + .source( + "test.py", + "from foo import __all__ as __all__ + TRICKSY = 1", + ) + .build(); + // `__all__` is imported from `foo.py` but it's + // empty, so `TRICKSY` is not part of the exported + // API. + insta::assert_snapshot!( + test.exports_for("test.py"), + @"", + ); + } + + #[test] + fn explicit_reexport_all_empty_then_added_to() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = [] + ", + ) + .source( + "test.py", + "from foo import __all__ + TRICKSY = 1 + __all__.append('TRICKSY')", + ) + .build(); + insta::assert_snapshot!( + test.exports_for("test.py"), + @"TRICKSY :: Constant", + ); + } + + #[test] + fn explicit_reexport_all_non_empty_but_non_existent() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = ['TRICKSY'] + ", + ) + .source("test.py", "from foo import __all__ as __all__") + .build(); + // `TRICKSY` is not a valid symbol, so it's not considered + // part of the exports of `test`. + insta::assert_snapshot!( + test.exports_for("test.py"), + @"", + ); + } + + #[test] + fn explicit_reexport_all_applies_to_importer() { + let test = PublicTestBuilder::default() + .source( + "foo.py", + " + ZQZQZQ = 1 + __all__ = ['TRICKSY'] + ", + ) + .source( + "test.py", + "from foo import __all__ + TRICKSY = 1", + ) + .build(); + insta::assert_snapshot!( + test.exports_for("test.py"), + @"TRICKSY :: Constant", + ); + } + fn matches(query: &str, symbol: &str) -> bool { super::QueryPattern::fuzzy(query).is_match_symbol_name(symbol) } diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap index cb2f8c55e3..2bdac4a17e 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap @@ -6,7 +6,7 @@ expression: completions { "label": "Literal (import typing)", "kind": 6, - "sortText": " 50", + "sortText": " 58", "insertText": "Literal", "additionalTextEdits": [ { @@ -27,7 +27,7 @@ expression: completions { "label": "LiteralString (import typing)", "kind": 6, - "sortText": " 51", + "sortText": " 59", "insertText": "LiteralString", "additionalTextEdits": [ { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap index cb2f8c55e3..2bdac4a17e 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap @@ -6,7 +6,7 @@ expression: completions { "label": "Literal (import typing)", "kind": 6, - "sortText": " 50", + "sortText": " 58", "insertText": "Literal", "additionalTextEdits": [ { @@ -27,7 +27,7 @@ expression: completions { "label": "LiteralString (import typing)", "kind": 6, - "sortText": " 51", + "sortText": " 59", "insertText": "LiteralString", "additionalTextEdits": [ { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap index cb2f8c55e3..2bdac4a17e 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap @@ -6,7 +6,7 @@ expression: completions { "label": "Literal (import typing)", "kind": 6, - "sortText": " 50", + "sortText": " 58", "insertText": "Literal", "additionalTextEdits": [ { @@ -27,7 +27,7 @@ expression: completions { "label": "LiteralString (import typing)", "kind": 6, - "sortText": " 51", + "sortText": " 59", "insertText": "LiteralString", "additionalTextEdits": [ { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap index b7d8c9907a..a0ff0b77b6 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap @@ -6,7 +6,7 @@ expression: completions { "label": "Literal (import typing)", "kind": 6, - "sortText": " 50", + "sortText": " 58", "insertText": "Literal", "additionalTextEdits": [ { @@ -27,7 +27,7 @@ expression: completions { "label": "LiteralString (import typing)", "kind": 6, - "sortText": " 51", + "sortText": " 59", "insertText": "LiteralString", "additionalTextEdits": [ { From 2a38395bc856dc741e4a991ea241d15c741ece4f Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Tue, 25 Nov 2025 14:02:59 -0500 Subject: [PATCH 21/41] [ty] Add some tests for re-exports and `__all__` to completions Note that the `Deprecated` symbols from `importlib.metadata` are no longer offered because 1) `importlib.metadata` defined `__all__` and 2) the `Deprecated` symbols aren't in it. These seem to not be a part of its public API according to the docs, so this seems right to me. --- crates/ty_ide/src/completion.rs | 128 ++++++++++++++++++++++++++++---- 1 file changed, 113 insertions(+), 15 deletions(-) diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index ae8e75cb9d..ba4c7e9e5b 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -5812,13 +5812,7 @@ from .imp let builder = completion_test_builder("deprecated") .auto_import() .module_names(); - assert_snapshot!(builder.build().snapshot(), @r" - Deprecated :: importlib.metadata - DeprecatedList :: importlib.metadata - DeprecatedNonAbstract :: importlib.metadata - DeprecatedTuple :: importlib.metadata - deprecated :: warnings - "); + assert_snapshot!(builder.build().snapshot(), @"deprecated :: warnings"); } #[test] @@ -5843,10 +5837,6 @@ from .imp .auto_import() .module_names(); assert_snapshot!(builder.build().snapshot(), @r" - Deprecated :: importlib.metadata - DeprecatedList :: importlib.metadata - DeprecatedNonAbstract :: importlib.metadata - DeprecatedTuple :: importlib.metadata deprecated :: typing_extensions deprecated :: warnings "); @@ -5872,15 +5862,123 @@ from .imp .auto_import() .module_names(); assert_snapshot!(builder.build().snapshot(), @r" - Deprecated :: importlib.metadata - DeprecatedList :: importlib.metadata - DeprecatedNonAbstract :: importlib.metadata - DeprecatedTuple :: importlib.metadata deprecated :: typing_extensions deprecated :: warnings "); } + #[test] + fn reexport_simple_import_noauto() { + let snapshot = CursorTest::builder() + .source( + "main.py", + r#" +import foo +foo.ZQ +"#, + ) + .source("foo.py", r#"from bar import ZQZQ"#) + .source("bar.py", r#"ZQZQ = 1"#) + .completion_test_builder() + .module_names() + .build() + .snapshot(); + assert_snapshot!(snapshot, @"ZQZQ :: Current module"); + } + + #[test] + fn reexport_simple_import_auto() { + let snapshot = CursorTest::builder() + .source( + "main.py", + r#" +ZQ +"#, + ) + .source("foo.py", r#"from bar import ZQZQ"#) + .source("bar.py", r#"ZQZQ = 1"#) + .completion_test_builder() + .auto_import() + .module_names() + .build() + .snapshot(); + // We're specifically looking for `ZQZQ` in `bar` + // here but *not* in `foo`. Namely, in `foo`, + // `ZQZQ` is a "regular" import that is not by + // convention considered a re-export. + assert_snapshot!(snapshot, @"ZQZQ :: bar"); + } + + #[test] + fn reexport_redundant_convention_import_noauto() { + let snapshot = CursorTest::builder() + .source( + "main.py", + r#" +import foo +foo.ZQ +"#, + ) + .source("foo.py", r#"from bar import ZQZQ as ZQZQ"#) + .source("bar.py", r#"ZQZQ = 1"#) + .completion_test_builder() + .module_names() + .build() + .snapshot(); + assert_snapshot!(snapshot, @"ZQZQ :: Current module"); + } + + #[test] + fn reexport_redundant_convention_import_auto() { + let snapshot = CursorTest::builder() + .source( + "main.py", + r#" +ZQ +"#, + ) + .source("foo.py", r#"from bar import ZQZQ as ZQZQ"#) + .source("bar.py", r#"ZQZQ = 1"#) + .completion_test_builder() + .auto_import() + .module_names() + .build() + .snapshot(); + assert_snapshot!(snapshot, @r" + ZQZQ :: bar + ZQZQ :: foo + "); + } + + #[test] + fn auto_import_respects_all() { + let snapshot = CursorTest::builder() + .source( + "main.py", + r#" +ZQ +"#, + ) + .source( + "bar.py", + r#" + ZQZQ1 = 1 + ZQZQ2 = 1 + __all__ = ['ZQZQ1'] + "#, + ) + .completion_test_builder() + .auto_import() + .module_names() + .build() + .snapshot(); + // We specifically do not want `ZQZQ2` here, since + // it is not part of `__all__`. + assert_snapshot!(snapshot, @r" + ZQZQ1 :: bar + "); + } + /// A way to create a simple single-file (named `main.py`) completion test /// builder. /// From 32f400a457677a2dc79bcb989065dba467fe1633 Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Wed, 3 Dec 2025 14:59:21 -0500 Subject: [PATCH 22/41] [ty] Make auto-import ignore symbols in modules starting with a `_` This applies recursively. So if *any* component of a module name starts with a `_`, then symbols from that module are excluded from auto-import. The exception is when it's a module within first party code. Then we want to include it in auto-import. --- crates/ty_ide/src/all_symbols.rs | 14 +++ crates/ty_ide/src/completion.rs | 88 +++++++++++++++++++ .../src/module_resolver/path.rs | 2 +- .../snapshots/e2e__notebook__auto_import.snap | 4 +- .../e2e__notebook__auto_import_docstring.snap | 4 +- ...2e__notebook__auto_import_from_future.snap | 4 +- .../e2e__notebook__auto_import_same_cell.snap | 4 +- 7 files changed, 111 insertions(+), 9 deletions(-) diff --git a/crates/ty_ide/src/all_symbols.rs b/crates/ty_ide/src/all_symbols.rs index c7282a5fc7..79767d36d1 100644 --- a/crates/ty_ide/src/all_symbols.rs +++ b/crates/ty_ide/src/all_symbols.rs @@ -36,6 +36,20 @@ pub fn all_symbols<'db>( let Some(file) = module.file(&*db) else { continue; }; + // By convention, modules starting with an underscore + // are generally considered unexported. However, we + // should consider first party modules fair game. + // + // Note that we apply this recursively. e.g., + // `numpy._core.multiarray` is considered private + // because it's a child of `_core`. + if module.name(&*db).components().any(|c| c.starts_with('_')) + && module + .search_path(&*db) + .is_none_or(|sp| !sp.is_first_party()) + { + continue; + } // TODO: also make it available in `TYPE_CHECKING` blocks // (we'd need https://github.com/astral-sh/ty/issues/1553 to do this well) if !is_typing_extensions_available && module.name(&*db) == &typing_extensions { diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index ba4c7e9e5b..d751611339 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -5979,6 +5979,94 @@ ZQ "); } + // This test confirms current behavior (as of 2025-12-04), but + // it's not consistent with auto-import. That is, it doesn't + // strictly respect `__all__` on `bar`, but perhaps it should. + // + // See: https://github.com/astral-sh/ty/issues/1757 + #[test] + fn object_attr_ignores_all() { + let snapshot = CursorTest::builder() + .source( + "main.py", + r#" +import bar +bar.ZQ +"#, + ) + .source( + "bar.py", + r#" + ZQZQ1 = 1 + ZQZQ2 = 1 + __all__ = ['ZQZQ1'] + "#, + ) + .completion_test_builder() + .auto_import() + .module_names() + .build() + .snapshot(); + // We specifically do not want `ZQZQ2` here, since + // it is not part of `__all__`. + assert_snapshot!(snapshot, @r" + ZQZQ1 :: + ZQZQ2 :: + "); + } + + #[test] + fn auto_import_ignores_modules_with_leading_underscore() { + let snapshot = CursorTest::builder() + .source( + "main.py", + r#" +Quitter +"#, + ) + .completion_test_builder() + .auto_import() + .module_names() + .build() + .snapshot(); + // There is a `Quitter` in `_sitebuiltins` in the standard + // library. But this is skipped by auto-import because it's + // 1) not first party and 2) starts with an `_`. + assert_snapshot!(snapshot, @""); + } + + #[test] + fn auto_import_includes_modules_with_leading_underscore_in_first_party() { + let snapshot = CursorTest::builder() + .source( + "main.py", + r#" +ZQ +"#, + ) + .source( + "bar.py", + r#" + ZQZQ1 = 1 + "#, + ) + .source( + "_foo.py", + r#" + ZQZQ1 = 1 + "#, + ) + .completion_test_builder() + .auto_import() + .module_names() + .build() + .snapshot(); + assert_snapshot!(snapshot, @r" + ZQZQ1 :: _foo + ZQZQ1 :: bar + "); + } + /// A way to create a simple single-file (named `main.py`) completion test /// builder. /// diff --git a/crates/ty_python_semantic/src/module_resolver/path.rs b/crates/ty_python_semantic/src/module_resolver/path.rs index 5200396dc1..638bbf819a 100644 --- a/crates/ty_python_semantic/src/module_resolver/path.rs +++ b/crates/ty_python_semantic/src/module_resolver/path.rs @@ -594,7 +594,7 @@ impl SearchPath { ) } - pub(crate) fn is_first_party(&self) -> bool { + pub fn is_first_party(&self) -> bool { matches!(&*self.0, SearchPathInner::FirstParty(_)) } diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap index 2bdac4a17e..772f80a795 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap @@ -6,7 +6,7 @@ expression: completions { "label": "Literal (import typing)", "kind": 6, - "sortText": " 58", + "sortText": " 35", "insertText": "Literal", "additionalTextEdits": [ { @@ -27,7 +27,7 @@ expression: completions { "label": "LiteralString (import typing)", "kind": 6, - "sortText": " 59", + "sortText": " 36", "insertText": "LiteralString", "additionalTextEdits": [ { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap index 2bdac4a17e..772f80a795 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap @@ -6,7 +6,7 @@ expression: completions { "label": "Literal (import typing)", "kind": 6, - "sortText": " 58", + "sortText": " 35", "insertText": "Literal", "additionalTextEdits": [ { @@ -27,7 +27,7 @@ expression: completions { "label": "LiteralString (import typing)", "kind": 6, - "sortText": " 59", + "sortText": " 36", "insertText": "LiteralString", "additionalTextEdits": [ { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap index 2bdac4a17e..772f80a795 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap @@ -6,7 +6,7 @@ expression: completions { "label": "Literal (import typing)", "kind": 6, - "sortText": " 58", + "sortText": " 35", "insertText": "Literal", "additionalTextEdits": [ { @@ -27,7 +27,7 @@ expression: completions { "label": "LiteralString (import typing)", "kind": 6, - "sortText": " 59", + "sortText": " 36", "insertText": "LiteralString", "additionalTextEdits": [ { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap index a0ff0b77b6..004f9f6823 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap @@ -6,7 +6,7 @@ expression: completions { "label": "Literal (import typing)", "kind": 6, - "sortText": " 58", + "sortText": " 35", "insertText": "Literal", "additionalTextEdits": [ { @@ -27,7 +27,7 @@ expression: completions { "label": "LiteralString (import typing)", "kind": 6, - "sortText": " 59", + "sortText": " 36", "insertText": "LiteralString", "additionalTextEdits": [ { From e154efa229f2474e3ab333e328cf2aac623baec4 Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Wed, 3 Dec 2025 15:26:30 -0500 Subject: [PATCH 23/41] [ty] Update evaluation results These are all improvements here with one slight regression on `reveal_type` ranking. The previous completions offered were: ``` $ cargo r -q -p ty_completion_eval show-one ty-extensions-lower-stdlib ENOTRECOVERABLE (module: errno) REG_WHOLE_HIVE_VOLATILE (module: winreg) SQLITE_NOTICE_RECOVER_WAL (module: _sqlite3) SupportsGetItemViewable (module: _typeshed) removeHandler (module: unittest.signals) reveal_mro (module: ty_extensions) reveal_protocol_interface (module: ty_extensions) reveal_type (module: typing) (*, 8/10) _remove_original_values (module: _osx_support) _remove_universal_flags (module: _osx_support) ----- found 10 completions ``` And now they are: ``` $ cargo r -q -p ty_completion_eval show-one ty-extensions-lower-stdlib ENOTRECOVERABLE (module: errno) REG_WHOLE_HIVE_VOLATILE (module: winreg) SQLITE_NOTICE_RECOVER_WAL (module: sqlite3) SQLITE_NOTICE_RECOVER_WAL (module: sqlite3.dbapi2) removeHandler (module: unittest) removeHandler (module: unittest.signals) reveal_mro (module: ty_extensions) reveal_protocol_interface (module: ty_extensions) reveal_type (module: typing) (*, 9/9) ----- found 9 completions ``` Some completions were removed (because they are now considered unexported) and some were added (likely do to better re-export support). This particular case probably warrants more special attention anyway. So I think this is fine. (It's only a one-ranking regression.) --- crates/ty_completion_eval/completion-evaluation-tasks.csv | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/ty_completion_eval/completion-evaluation-tasks.csv b/crates/ty_completion_eval/completion-evaluation-tasks.csv index f8347cb8e5..a196fb98e4 100644 --- a/crates/ty_completion_eval/completion-evaluation-tasks.csv +++ b/crates/ty_completion_eval/completion-evaluation-tasks.csv @@ -11,9 +11,9 @@ import-deprioritizes-type_check_only,main.py,2,1 import-deprioritizes-type_check_only,main.py,3,2 import-deprioritizes-type_check_only,main.py,4,3 import-keyword-completion,main.py,0,1 -internal-typeshed-hidden,main.py,0,4 +internal-typeshed-hidden,main.py,0,2 none-completion,main.py,0,2 -numpy-array,main.py,0, +numpy-array,main.py,0,159 numpy-array,main.py,1,1 object-attr-instance-methods,main.py,0,1 object-attr-instance-methods,main.py,1,1 @@ -23,6 +23,6 @@ scope-existing-over-new-import,main.py,0,1 scope-prioritize-closer,main.py,0,2 scope-simple-long-identifier,main.py,0,1 tstring-completions,main.py,0,1 -ty-extensions-lower-stdlib,main.py,0,8 +ty-extensions-lower-stdlib,main.py,0,9 type-var-typing-over-ast,main.py,0,3 -type-var-typing-over-ast,main.py,1,275 +type-var-typing-over-ast,main.py,1,239 From f054e7edf83632637651b4f77020f0cd7497d76c Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Thu, 4 Dec 2025 09:13:17 -0500 Subject: [PATCH 24/41] [ty] Tweaks tests to use clearer language A completion lacking a module reference doesn't necessarily mean that the symbol is defined within the current module. I believe the intent here is that it means that no import is required to use it. --- crates/ty_ide/src/completion.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index d751611339..cd4e886e45 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -4350,7 +4350,7 @@ from os. .build() .snapshot(); assert_snapshot!(snapshot, @r" - Kadabra :: Literal[1] :: Current module + Kadabra :: Literal[1] :: AbraKadabra :: Unavailable :: package "); } @@ -5534,7 +5534,7 @@ def foo(param: s) // Even though long_namea is alphabetically before long_nameb, // long_nameb is currently imported and should be preferred. assert_snapshot!(snapshot, @r" - long_nameb :: Literal[1] :: Current module + long_nameb :: Literal[1] :: long_namea :: Unavailable :: foo "); } @@ -5804,7 +5804,7 @@ from .imp #[test] fn typing_extensions_excluded_from_import() { let builder = completion_test_builder("from typing").module_names(); - assert_snapshot!(builder.build().snapshot(), @"typing :: Current module"); + assert_snapshot!(builder.build().snapshot(), @"typing :: "); } #[test] @@ -5823,8 +5823,8 @@ from .imp .completion_test_builder() .module_names(); assert_snapshot!(builder.build().snapshot(), @r" - typing :: Current module - typing_extensions :: Current module + typing :: + typing_extensions :: "); } @@ -5849,8 +5849,8 @@ from .imp .completion_test_builder() .module_names(); assert_snapshot!(builder.build().snapshot(), @r" - typing :: Current module - typing_extensions :: Current module + typing :: + typing_extensions :: "); } @@ -5883,7 +5883,7 @@ foo.ZQ .module_names() .build() .snapshot(); - assert_snapshot!(snapshot, @"ZQZQ :: Current module"); + assert_snapshot!(snapshot, @"ZQZQ :: "); } #[test] @@ -5925,7 +5925,7 @@ foo.ZQ .module_names() .build() .snapshot(); - assert_snapshot!(snapshot, @"ZQZQ :: Current module"); + assert_snapshot!(snapshot, @"ZQZQ :: "); } #[test] @@ -6241,7 +6241,7 @@ ZQ let module_name = c .module_name .map(ModuleName::as_str) - .unwrap_or("Current module"); + .unwrap_or(""); snapshot = format!("{snapshot} :: {module_name}"); } snapshot From 6a025d1925c9e5eb4b8e74ef1925dca3d2d7130f Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Thu, 4 Dec 2025 12:08:10 -0500 Subject: [PATCH 25/41] [ty] Redact ranking of completions from e2e LSP tests I think changes to this value are generally noise. It's hard to tell what it means and it isn't especially actionable. We already have an eval running in CI for completion ranking, so I don't think it's terribly important to care about ranking here in e2e tests _generally_. --- crates/ty_server/tests/e2e/notebook.rs | 26 ++++++++++++++++--- .../snapshots/e2e__notebook__auto_import.snap | 4 +-- .../e2e__notebook__auto_import_docstring.snap | 4 +-- ...2e__notebook__auto_import_from_future.snap | 4 +-- .../e2e__notebook__auto_import_same_cell.snap | 4 +-- 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/crates/ty_server/tests/e2e/notebook.rs b/crates/ty_server/tests/e2e/notebook.rs index 4deb2bed17..b8cb10643b 100644 --- a/crates/ty_server/tests/e2e/notebook.rs +++ b/crates/ty_server/tests/e2e/notebook.rs @@ -5,6 +5,8 @@ use ty_server::ClientOptions; use crate::{TestServer, TestServerBuilder}; +static FILTERS: &[(&str, &str)] = &[(r#""sortText": "[0-9 ]+""#, r#""sortText": "[RANKING]""#)]; + #[test] fn publish_diagnostics_open() -> anyhow::Result<()> { let mut server = TestServerBuilder::new()? @@ -309,7 +311,11 @@ b: Litera let completions = literal_completions(&mut server, &second_cell, Position::new(1, 9)); - assert_json_snapshot!(completions); + insta::with_settings!({ + filters => FILTERS.iter().copied(), + }, { + assert_json_snapshot!(completions); + }); Ok(()) } @@ -340,7 +346,11 @@ b: Litera let completions = literal_completions(&mut server, &first_cell, Position::new(1, 9)); - assert_json_snapshot!(completions); + insta::with_settings!({ + filters => FILTERS.iter().copied(), + }, { + assert_json_snapshot!(completions); + }); Ok(()) } @@ -373,7 +383,11 @@ b: Litera let completions = literal_completions(&mut server, &second_cell, Position::new(1, 9)); - assert_json_snapshot!(completions); + insta::with_settings!({ + filters => FILTERS.iter().copied(), + }, { + assert_json_snapshot!(completions); + }); Ok(()) } @@ -409,7 +423,11 @@ b: Litera let completions = literal_completions(&mut server, &second_cell, Position::new(1, 9)); - assert_json_snapshot!(completions); + insta::with_settings!({ + filters => FILTERS.iter().copied(), + }, { + assert_json_snapshot!(completions); + }); Ok(()) } diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap index 772f80a795..a9740b7b97 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import.snap @@ -6,7 +6,7 @@ expression: completions { "label": "Literal (import typing)", "kind": 6, - "sortText": " 35", + "sortText": "[RANKING]", "insertText": "Literal", "additionalTextEdits": [ { @@ -27,7 +27,7 @@ expression: completions { "label": "LiteralString (import typing)", "kind": 6, - "sortText": " 36", + "sortText": "[RANKING]", "insertText": "LiteralString", "additionalTextEdits": [ { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap index 772f80a795..a9740b7b97 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_docstring.snap @@ -6,7 +6,7 @@ expression: completions { "label": "Literal (import typing)", "kind": 6, - "sortText": " 35", + "sortText": "[RANKING]", "insertText": "Literal", "additionalTextEdits": [ { @@ -27,7 +27,7 @@ expression: completions { "label": "LiteralString (import typing)", "kind": 6, - "sortText": " 36", + "sortText": "[RANKING]", "insertText": "LiteralString", "additionalTextEdits": [ { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap index 772f80a795..a9740b7b97 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_from_future.snap @@ -6,7 +6,7 @@ expression: completions { "label": "Literal (import typing)", "kind": 6, - "sortText": " 35", + "sortText": "[RANKING]", "insertText": "Literal", "additionalTextEdits": [ { @@ -27,7 +27,7 @@ expression: completions { "label": "LiteralString (import typing)", "kind": 6, - "sortText": " 36", + "sortText": "[RANKING]", "insertText": "LiteralString", "additionalTextEdits": [ { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap index 004f9f6823..713c26841e 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__notebook__auto_import_same_cell.snap @@ -6,7 +6,7 @@ expression: completions { "label": "Literal (import typing)", "kind": 6, - "sortText": " 35", + "sortText": "[RANKING]", "insertText": "Literal", "additionalTextEdits": [ { @@ -27,7 +27,7 @@ expression: completions { "label": "LiteralString (import typing)", "kind": 6, - "sortText": " 36", + "sortText": "[RANKING]", "insertText": "LiteralString", "additionalTextEdits": [ { From fdcb5a7e731c934d2820cef74b60c16a838a9c97 Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Thu, 4 Dec 2025 12:23:59 -0500 Subject: [PATCH 26/41] [ty] Clarify the use of `SymbolKind` in auto-import --- crates/ty_ide/src/symbols.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/ty_ide/src/symbols.rs b/crates/ty_ide/src/symbols.rs index a80a9ed56d..ba33af9ef2 100644 --- a/crates/ty_ide/src/symbols.rs +++ b/crates/ty_ide/src/symbols.rs @@ -292,7 +292,13 @@ impl<'a> From<&'a SymbolTreeWithChildren> for SymbolInfo<'a> { } } -/// The kind of symbol +/// The kind of symbol. +/// +/// Note that this is computed on a best effort basis. The nature of +/// auto-import is that it tries to do a very low effort scan of a lot of code +/// very quickly. This means that it doesn't use things like type information +/// or completely resolve the definition of every symbol. So for example, we +/// might label a module as a variable, depending on how it was introduced. #[derive(Debug, Clone, Copy, PartialEq, Eq, get_size2::GetSize)] pub enum SymbolKind { Module, From 3c2cf49f603a10538cdbfdac1d0ff9b3efb46b00 Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Thu, 4 Dec 2025 13:58:15 -0500 Subject: [PATCH 27/41] [ty] Refactor auto-import symbol info This just encapsulates the representation so that we can make changes to it more easily. --- crates/ty_ide/src/all_symbols.rs | 40 ++++++++++++++++++++++++++++---- crates/ty_ide/src/completion.rs | 11 ++++----- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/crates/ty_ide/src/all_symbols.rs b/crates/ty_ide/src/all_symbols.rs index 79767d36d1..210b116e57 100644 --- a/crates/ty_ide/src/all_symbols.rs +++ b/crates/ty_ide/src/all_symbols.rs @@ -2,7 +2,10 @@ use ruff_db::files::File; use ty_project::Db; use ty_python_semantic::{Module, ModuleName, all_modules, resolve_real_shadowable_module}; -use crate::symbols::{QueryPattern, SymbolInfo, symbols_for_file_global_only}; +use crate::{ + SymbolKind, + symbols::{QueryPattern, SymbolInfo, symbols_for_file_global_only}, +}; /// Get all symbols matching the query string. /// @@ -85,14 +88,43 @@ pub fn all_symbols<'db>( #[derive(Debug, Clone, PartialEq, Eq)] pub struct AllSymbolInfo<'db> { /// The symbol information. - pub symbol: SymbolInfo<'static>, + symbol: SymbolInfo<'static>, /// The module containing the symbol. - pub module: Module<'db>, + module: Module<'db>, /// The file containing the symbol. /// /// This `File` is guaranteed to be the same /// as the `File` underlying `module`. - pub file: File, + file: File, +} + +impl<'db> AllSymbolInfo<'db> { + /// Returns the name of this symbol. + pub fn name(&self) -> &str { + &self.symbol.name + } + + /// Returns the "kind" of this symbol. + /// + /// The kind of a symbol in the context of auto-import is + /// determined on a best effort basis. It may be imprecise + /// in some cases, e.g., reporting a module as a variable. + pub fn kind(&self) -> SymbolKind { + self.symbol.kind + } + + /// Returns the module this symbol is exported from. + pub fn module(&self) -> Module<'db> { + self.module + } + + /// Returns the `File` corresponding to the module. + /// + /// This is always equivalent to + /// `AllSymbolInfo::module().file().unwrap()`. + pub fn file(&self) -> File { + self.file + } } #[cfg(test)] diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index cd4e886e45..f96ac1500d 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -537,12 +537,11 @@ fn add_unimported_completions<'db>( let members = importer.members_in_scope_at(scoped.node, scoped.node.start()); for symbol in all_symbols(db, file, &completions.query) { - if symbol.module.file(db) == Some(file) || symbol.module.is_known(db, KnownModule::Builtins) - { + if symbol.file() == file || symbol.module().is_known(db, KnownModule::Builtins) { continue; } - let request = create_import_request(symbol.module.name(db), &symbol.symbol.name); + let request = create_import_request(symbol.module().name(db), symbol.name()); // FIXME: `all_symbols` doesn't account for wildcard imports. // Since we're looking at every module, this is probably // "fine," but it might mean that we import a symbol from the @@ -551,11 +550,11 @@ fn add_unimported_completions<'db>( // N.B. We use `add` here because `all_symbols` already // takes our query into account. completions.force_add(Completion { - name: ast::name::Name::new(&symbol.symbol.name), + name: ast::name::Name::new(symbol.name()), insert: Some(import_action.symbol_text().into()), ty: None, - kind: symbol.symbol.kind.to_completion_kind(), - module_name: Some(symbol.module.name(db)), + kind: symbol.kind().to_completion_kind(), + module_name: Some(symbol.module().name(db)), import: import_action.import().cloned(), builtin: false, // TODO: `is_type_check_only` requires inferring the type of the symbol From da94b992485ccb6f473df1df9e1f4af2ff1eb948 Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Thu, 4 Dec 2025 14:33:21 -0500 Subject: [PATCH 28/41] [ty] Add support for module-only import requests The existing importer functionality always required an import request with a module and a member in that module. But we want to be able to insert import statements for a module itself and not any members in the module. This is basically changing `member: &str` to an `Option<&str>` and fixing the fallout in a way that makes sense for module-only imports. --- crates/ty_ide/src/importer.rs | 150 ++++++++++++++++++++++++++++++---- 1 file changed, 134 insertions(+), 16 deletions(-) diff --git a/crates/ty_ide/src/importer.rs b/crates/ty_ide/src/importer.rs index 5ff46a1ae1..1dff46bcaf 100644 --- a/crates/ty_ide/src/importer.rs +++ b/crates/ty_ide/src/importer.rs @@ -145,7 +145,7 @@ impl<'a> Importer<'a> { members: &MembersInScope, ) -> ImportAction { let request = request.avoid_conflicts(self.db, self.file, members); - let mut symbol_text: Box = request.member.into(); + let mut symbol_text: Box = request.member.unwrap_or(request.module).into(); let Some(response) = self.find(&request, members.at) else { let insertion = if let Some(future) = self.find_last_future_import(members.at) { Insertion::end_of_statement(future.stmt, self.source, self.stylist) @@ -157,14 +157,27 @@ impl<'a> Importer<'a> { Insertion::start_of_file(self.parsed.suite(), self.source, self.stylist, range) }; let import = insertion.into_edit(&request.to_string()); - if matches!(request.style, ImportStyle::Import) { - symbol_text = format!("{}.{}", request.module, request.member).into(); + if let Some(member) = request.member + && matches!(request.style, ImportStyle::Import) + { + symbol_text = format!("{}.{}", request.module, member).into(); } return ImportAction { import: Some(import), symbol_text, }; }; + + // When we just have a request to import a module (and not + // any members from that module), then the only way we can be + // here is if we found a pre-existing import that definitively + // satisfies the request. So we're done. + let Some(member) = request.member else { + return ImportAction { + import: None, + symbol_text, + }; + }; match response.kind { ImportResponseKind::Unqualified { ast, alias } => { let member = alias.asname.as_ref().unwrap_or(&alias.name).as_str(); @@ -189,13 +202,10 @@ impl<'a> Importer<'a> { let import = if let Some(insertion) = Insertion::existing_import(response.import.stmt, self.tokens) { - insertion.into_edit(request.member) + insertion.into_edit(member) } else { Insertion::end_of_statement(response.import.stmt, self.source, self.stylist) - .into_edit(&format!( - "from {} import {}", - request.module, request.member - )) + .into_edit(&format!("from {} import {member}", request.module)) }; ImportAction { import: Some(import), @@ -481,6 +491,17 @@ impl<'ast> AstImportKind<'ast> { Some(ImportResponseKind::Qualified { ast, alias }) } AstImportKind::ImportFrom(ast) => { + // If the request is for a module itself, then we + // assume that it can never be satisfies by a + // `from ... import ...` statement. For example, a + // `request for collections.abc` needs an + // `import collections.abc`. Now, there could be a + // `from collections import abc`, and we could + // plausibly consider that a match and return a + // symbol text of `abc`. But it's not clear if that's + // the right choice or not. + let member = request.member?; + if request.force_style && !matches!(request.style, ImportStyle::ImportFrom) { return None; } @@ -492,9 +513,7 @@ impl<'ast> AstImportKind<'ast> { let kind = ast .names .iter() - .find(|alias| { - alias.name.as_str() == "*" || alias.name.as_str() == request.member - }) + .find(|alias| alias.name.as_str() == "*" || alias.name.as_str() == member) .map(|alias| ImportResponseKind::Unqualified { ast, alias }) .unwrap_or_else(|| ImportResponseKind::Partial(ast)); Some(kind) @@ -510,7 +529,10 @@ pub(crate) struct ImportRequest<'a> { /// `foo`, in `from foo import bar`). module: &'a str, /// The member to import (e.g., `bar`, in `from foo import bar`). - member: &'a str, + /// + /// When `member` is absent, then this request reflects an import + /// of the module itself. i.e., `import module`. + member: Option<&'a str>, /// The preferred style to use when importing the symbol (e.g., /// `import foo` or `from foo import bar`). /// @@ -532,7 +554,7 @@ impl<'a> ImportRequest<'a> { pub(crate) fn import(module: &'a str, member: &'a str) -> Self { Self { module, - member, + member: Some(member), style: ImportStyle::Import, force_style: false, } @@ -545,12 +567,26 @@ impl<'a> ImportRequest<'a> { pub(crate) fn import_from(module: &'a str, member: &'a str) -> Self { Self { module, - member, + member: Some(member), style: ImportStyle::ImportFrom, force_style: false, } } + /// Create a new [`ImportRequest`] for bringing the given module + /// into scope. + /// + /// This is for just importing the module itself, always via an + /// `import` statement. + pub(crate) fn module(module: &'a str) -> Self { + Self { + module, + member: None, + style: ImportStyle::Import, + force_style: false, + } + } + /// Causes this request to become a command. This will force the /// requested import style, even if another style would be more /// appropriate generally. @@ -565,7 +601,13 @@ impl<'a> ImportRequest<'a> { /// of an import conflict are minimized (although not always reduced /// to zero). fn avoid_conflicts(self, db: &dyn Db, importing_file: File, members: &MembersInScope) -> Self { - match (members.map.get(self.module), members.map.get(self.member)) { + let Some(member) = self.member else { + return Self { + style: ImportStyle::Import, + ..self + }; + }; + match (members.map.get(self.module), members.map.get(member)) { // Neither symbol exists, so we can just proceed as // normal. (None, None) => self, @@ -630,7 +672,10 @@ impl std::fmt::Display for ImportRequest<'_> { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self.style { ImportStyle::Import => write!(f, "import {}", self.module), - ImportStyle::ImportFrom => write!(f, "from {} import {}", self.module, self.member), + ImportStyle::ImportFrom => match self.member { + None => write!(f, "import {}", self.module), + Some(member) => write!(f, "from {} import {member}", self.module), + }, } } } @@ -843,6 +888,10 @@ mod tests { self.add(ImportRequest::import_from(module, member)) } + fn module(&self, module: &str) -> String { + self.add(ImportRequest::module(module)) + } + fn add(&self, request: ImportRequest<'_>) -> String { let node = covering_node( self.cursor.parsed.syntax().into(), @@ -2156,4 +2205,73 @@ except ImportError: (bar.MAGIC) "); } + + #[test] + fn import_module_blank() { + let test = cursor_test( + "\ + + ", + ); + assert_snapshot!( + test.module("collections"), @r" + import collections + collections + "); + } + + #[test] + fn import_module_exists() { + let test = cursor_test( + "\ +import collections + + ", + ); + assert_snapshot!( + test.module("collections"), @r" + import collections + collections + "); + } + + #[test] + fn import_module_from_exists() { + let test = cursor_test( + "\ +from collections import defaultdict + + ", + ); + assert_snapshot!( + test.module("collections"), @r" + import collections + from collections import defaultdict + collections + "); + } + + // This test is working as intended. That is, + // `abc` is already in scope, so requesting an + // import for `collections.abc` could feasibly + // reuse the import and rewrite the symbol text + // to just `abc`. But for now it seems better + // to respect what has been written and add the + // `import collections.abc`. This behavior could + // plausibly be changed. + #[test] + fn import_module_from_via_member_exists() { + let test = cursor_test( + "\ +from collections import abc + + ", + ); + assert_snapshot!( + test.module("collections.abc"), @r" + import collections.abc + from collections import abc + collections.abc + "); + } } From 518d11b33f0f7acc48d9e79c15728dedd6e25239 Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Thu, 4 Dec 2025 14:33:42 -0500 Subject: [PATCH 29/41] [ty] Add modules to auto-import This makes auto-import include modules in suggestions. In this initial implementation, we permit this to include submodules as well. This is in contrast to what we do in `import ...` completions. It's easy to change this behavior, but I think it'd be interesting to run with this for now to see how well it works. --- crates/ty_ide/src/all_symbols.rs | 74 +++++++++---- crates/ty_ide/src/completion.rs | 100 +++++++++++++++++- ...action_attribute_access_on_unimported.snap | 46 ++++++++ 3 files changed, 196 insertions(+), 24 deletions(-) diff --git a/crates/ty_ide/src/all_symbols.rs b/crates/ty_ide/src/all_symbols.rs index 210b116e57..aa7f9e02b7 100644 --- a/crates/ty_ide/src/all_symbols.rs +++ b/crates/ty_ide/src/all_symbols.rs @@ -59,12 +59,19 @@ pub fn all_symbols<'db>( continue; } s.spawn(move |_| { + if query.is_match_symbol_name(module.name(&*db)) { + results.lock().unwrap().push(AllSymbolInfo { + symbol: None, + module, + file, + }); + } for (_, symbol) in symbols_for_file_global_only(&*db, file).search(query) { // It seems like we could do better here than // locking `results` for every single symbol, // but this works pretty well as it is. results.lock().unwrap().push(AllSymbolInfo { - symbol: symbol.to_owned(), + symbol: Some(symbol.to_owned()), module, file, }); @@ -76,8 +83,16 @@ pub fn all_symbols<'db>( let mut results = results.into_inner().unwrap(); results.sort_by(|s1, s2| { - let key1 = (&s1.symbol.name, s1.file.path(db).as_str()); - let key2 = (&s2.symbol.name, s2.file.path(db).as_str()); + let key1 = ( + s1.name_in_file() + .unwrap_or_else(|| s1.module().name(db).as_str()), + s1.file.path(db).as_str(), + ); + let key2 = ( + s2.name_in_file() + .unwrap_or_else(|| s2.module().name(db).as_str()), + s2.file.path(db).as_str(), + ); key1.cmp(&key2) }); results @@ -88,7 +103,9 @@ pub fn all_symbols<'db>( #[derive(Debug, Clone, PartialEq, Eq)] pub struct AllSymbolInfo<'db> { /// The symbol information. - symbol: SymbolInfo<'static>, + /// + /// When absent, this implies the symbol is the module itself. + symbol: Option>, /// The module containing the symbol. module: Module<'db>, /// The file containing the symbol. @@ -99,9 +116,14 @@ pub struct AllSymbolInfo<'db> { } impl<'db> AllSymbolInfo<'db> { - /// Returns the name of this symbol. - pub fn name(&self) -> &str { - &self.symbol.name + /// Returns the name of this symbol as it exists in a file. + /// + /// When absent, there is no concrete symbol in a module + /// somewhere. Instead, this represents importing a module. + /// In this case, if the caller needs a symbol name, they + /// should use `AllSymbolInfo::module().name()`. + pub fn name_in_file(&self) -> Option<&str> { + self.symbol.as_ref().map(|symbol| &*symbol.name) } /// Returns the "kind" of this symbol. @@ -110,7 +132,10 @@ impl<'db> AllSymbolInfo<'db> { /// determined on a best effort basis. It may be imprecise /// in some cases, e.g., reporting a module as a variable. pub fn kind(&self) -> SymbolKind { - self.symbol.kind + self.symbol + .as_ref() + .map(|symbol| symbol.kind) + .unwrap_or(SymbolKind::Module) } /// Returns the module this symbol is exported from. @@ -208,25 +233,31 @@ ABCDEFGHIJKLMNOP = 'https://api.example.com' return "No symbols found".to_string(); } - self.render_diagnostics(symbols.into_iter().map(AllSymbolDiagnostic::new)) + self.render_diagnostics(symbols.into_iter().map(|symbol_info| AllSymbolDiagnostic { + db: &self.db, + symbol_info, + })) } } struct AllSymbolDiagnostic<'db> { + db: &'db dyn Db, symbol_info: AllSymbolInfo<'db>, } - impl<'db> AllSymbolDiagnostic<'db> { - fn new(symbol_info: AllSymbolInfo<'db>) -> Self { - Self { symbol_info } - } - } - impl IntoDiagnostic for AllSymbolDiagnostic<'_> { fn into_diagnostic(self) -> Diagnostic { - let symbol_kind_str = self.symbol_info.symbol.kind.to_string(); + let symbol_kind_str = self.symbol_info.kind().to_string(); - let info_text = format!("{} {}", symbol_kind_str, self.symbol_info.symbol.name); + let info_text = format!( + "{} {}", + symbol_kind_str, + self.symbol_info.name_in_file().unwrap_or_else(|| self + .symbol_info + .module() + .name(self.db) + .as_str()) + ); let sub = SubDiagnostic::new(SubDiagnosticSeverity::Info, info_text); @@ -235,9 +266,12 @@ ABCDEFGHIJKLMNOP = 'https://api.example.com' Severity::Info, "AllSymbolInfo".to_string(), ); - main.annotate(Annotation::primary( - Span::from(self.symbol_info.file).with_range(self.symbol_info.symbol.name_range), - )); + + let mut span = Span::from(self.symbol_info.file()); + if let Some(ref symbol) = self.symbol_info.symbol { + span = span.with_range(symbol.name_range); + } + main.annotate(Annotation::primary(span)); main.sub(sub); main diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index f96ac1500d..70505ac4c8 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -74,7 +74,7 @@ impl<'db> Completions<'db> { .into_iter() .filter_map(|item| { Some(ImportEdit { - label: format!("import {}.{}", item.module_name?, item.name), + label: format!("import {}", item.qualified?), edit: item.import?, }) }) @@ -160,6 +160,10 @@ impl<'db> Extend> for Completions<'db> { pub struct Completion<'db> { /// The label shown to the user for this suggestion. pub name: Name, + /// The fully qualified name, when available. + /// + /// This is only set when `module_name` is available. + pub qualified: Option, /// The text that should be inserted at the cursor /// when the completion is selected. /// @@ -225,6 +229,7 @@ impl<'db> Completion<'db> { let is_type_check_only = semantic.is_type_check_only(db); Completion { name: semantic.name, + qualified: None, insert: None, ty: semantic.ty, kind: None, @@ -306,6 +311,7 @@ impl<'db> Completion<'db> { fn keyword(name: &str) -> Self { Completion { name: name.into(), + qualified: None, insert: None, ty: None, kind: Some(CompletionKind::Keyword), @@ -321,6 +327,7 @@ impl<'db> Completion<'db> { fn value_keyword(name: &str, ty: Type<'db>) -> Completion<'db> { Completion { name: name.into(), + qualified: None, insert: None, ty: Some(ty), kind: Some(CompletionKind::Keyword), @@ -541,7 +548,18 @@ fn add_unimported_completions<'db>( continue; } - let request = create_import_request(symbol.module().name(db), symbol.name()); + let module_name = symbol.module().name(db); + let (name, qualified, request) = symbol + .name_in_file() + .map(|name| { + let qualified = format!("{module_name}.{name}"); + (name, qualified, create_import_request(module_name, name)) + }) + .unwrap_or_else(|| { + let name = module_name.as_str(); + let qualified = name.to_string(); + (name, qualified, ImportRequest::module(name)) + }); // FIXME: `all_symbols` doesn't account for wildcard imports. // Since we're looking at every module, this is probably // "fine," but it might mean that we import a symbol from the @@ -550,11 +568,12 @@ fn add_unimported_completions<'db>( // N.B. We use `add` here because `all_symbols` already // takes our query into account. completions.force_add(Completion { - name: ast::name::Name::new(symbol.name()), + name: ast::name::Name::new(name), + qualified: Some(ast::name::Name::new(qualified)), insert: Some(import_action.symbol_text().into()), ty: None, kind: symbol.kind().to_completion_kind(), - module_name: Some(symbol.module().name(db)), + module_name: Some(module_name), import: import_action.import().cloned(), builtin: false, // TODO: `is_type_check_only` requires inferring the type of the symbol @@ -6066,6 +6085,79 @@ ZQ "); } + #[test] + fn auto_import_includes_stdlib_modules_as_suggestions() { + let snapshot = CursorTest::builder() + .source( + "main.py", + r#" +multiprocess +"#, + ) + .completion_test_builder() + .auto_import() + .build() + .snapshot(); + assert_snapshot!(snapshot, @r" + multiprocessing + multiprocessing.connection + multiprocessing.context + multiprocessing.dummy + multiprocessing.dummy.connection + multiprocessing.forkserver + multiprocessing.heap + multiprocessing.managers + multiprocessing.pool + multiprocessing.popen_fork + multiprocessing.popen_forkserver + multiprocessing.popen_spawn_posix + multiprocessing.popen_spawn_win32 + multiprocessing.process + multiprocessing.queues + multiprocessing.reduction + multiprocessing.resource_sharer + multiprocessing.resource_tracker + multiprocessing.shared_memory + multiprocessing.sharedctypes + multiprocessing.spawn + multiprocessing.synchronize + multiprocessing.util + "); + } + + #[test] + fn auto_import_includes_first_party_modules_as_suggestions() { + let snapshot = CursorTest::builder() + .source( + "main.py", + r#" +zqzqzq +"#, + ) + .source("zqzqzqzqzq.py", "") + .completion_test_builder() + .auto_import() + .build() + .snapshot(); + assert_snapshot!(snapshot, @"zqzqzqzqzq"); + } + + #[test] + fn auto_import_includes_sub_modules_as_suggestions() { + let snapshot = CursorTest::builder() + .source( + "main.py", + r#" +collabc +"#, + ) + .completion_test_builder() + .auto_import() + .build() + .snapshot(); + assert_snapshot!(snapshot, @"collections.abc"); + } + /// A way to create a simple single-file (named `main.py`) completion test /// builder. /// diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap index c82a14bc8e..3026696d8e 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__code_actions__code_action_attribute_access_on_unimported.snap @@ -3,6 +3,52 @@ source: crates/ty_server/tests/e2e/code_actions.rs expression: code_actions --- [ + { + "title": "import typing", + "kind": "quickfix", + "diagnostics": [ + { + "range": { + "start": { + "line": 0, + "character": 3 + }, + "end": { + "line": 0, + "character": 9 + } + }, + "severity": 1, + "code": "unresolved-reference", + "codeDescription": { + "href": "https://ty.dev/rules#unresolved-reference" + }, + "source": "ty", + "message": "Name `typing` used when not defined", + "relatedInformation": [] + } + ], + "edit": { + "changes": { + "file:///src/foo.py": [ + { + "range": { + "start": { + "line": 0, + "character": 0 + }, + "end": { + "line": 0, + "character": 0 + } + }, + "newText": "import typing\n" + } + ] + } + }, + "isPreferred": true + }, { "title": "Ignore 'unresolved-reference' for this line", "kind": "quickfix", From 06415b1877cb9d9c5a9d6eadcb42b1b9304f12cf Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Thu, 4 Dec 2025 15:07:01 -0500 Subject: [PATCH 30/41] [ty] Update completion eval to include modules Our parsing and confirming of symbol names is highly suspect, but I think it's fine for now. --- .../completion-evaluation-tasks.csv | 5 ++++- crates/ty_completion_eval/src/main.rs | 12 ++++++++++++ .../auto-import-includes-modules/completion.toml | 2 ++ .../truth/auto-import-includes-modules/main.py | 3 +++ .../auto-import-includes-modules/pyproject.toml | 5 +++++ .../truth/auto-import-includes-modules/uv.lock | 8 ++++++++ 6 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 crates/ty_completion_eval/truth/auto-import-includes-modules/completion.toml create mode 100644 crates/ty_completion_eval/truth/auto-import-includes-modules/main.py create mode 100644 crates/ty_completion_eval/truth/auto-import-includes-modules/pyproject.toml create mode 100644 crates/ty_completion_eval/truth/auto-import-includes-modules/uv.lock diff --git a/crates/ty_completion_eval/completion-evaluation-tasks.csv b/crates/ty_completion_eval/completion-evaluation-tasks.csv index a196fb98e4..ed036b44f6 100644 --- a/crates/ty_completion_eval/completion-evaluation-tasks.csv +++ b/crates/ty_completion_eval/completion-evaluation-tasks.csv @@ -1,4 +1,7 @@ name,file,index,rank +auto-import-includes-modules,main.py,0,1 +auto-import-includes-modules,main.py,1,7 +auto-import-includes-modules,main.py,2,1 auto-import-skips-current-module,main.py,0,1 fstring-completions,main.py,0,1 higher-level-symbols-preferred,main.py,0, @@ -25,4 +28,4 @@ scope-simple-long-identifier,main.py,0,1 tstring-completions,main.py,0,1 ty-extensions-lower-stdlib,main.py,0,9 type-var-typing-over-ast,main.py,0,3 -type-var-typing-over-ast,main.py,1,239 +type-var-typing-over-ast,main.py,1,251 diff --git a/crates/ty_completion_eval/src/main.rs b/crates/ty_completion_eval/src/main.rs index 146041a278..5d3b44ad18 100644 --- a/crates/ty_completion_eval/src/main.rs +++ b/crates/ty_completion_eval/src/main.rs @@ -506,9 +506,21 @@ struct CompletionAnswer { impl CompletionAnswer { /// Returns true when this answer matches the completion given. fn matches(&self, completion: &Completion) -> bool { + if let Some(ref qualified) = completion.qualified { + if qualified.as_str() == self.qualified() { + return true; + } + } self.symbol == completion.name.as_str() && self.module.as_deref() == completion.module_name.map(ModuleName::as_str) } + + fn qualified(&self) -> String { + self.module + .as_ref() + .map(|module| format!("{module}.{}", self.symbol)) + .unwrap_or_else(|| self.symbol.clone()) + } } /// Copy the Python project from `src_dir` to `dst_dir`. diff --git a/crates/ty_completion_eval/truth/auto-import-includes-modules/completion.toml b/crates/ty_completion_eval/truth/auto-import-includes-modules/completion.toml new file mode 100644 index 0000000000..cbd5805f07 --- /dev/null +++ b/crates/ty_completion_eval/truth/auto-import-includes-modules/completion.toml @@ -0,0 +1,2 @@ +[settings] +auto-import = true diff --git a/crates/ty_completion_eval/truth/auto-import-includes-modules/main.py b/crates/ty_completion_eval/truth/auto-import-includes-modules/main.py new file mode 100644 index 0000000000..a019ea5d71 --- /dev/null +++ b/crates/ty_completion_eval/truth/auto-import-includes-modules/main.py @@ -0,0 +1,3 @@ +multiprocess +collect +collabc diff --git a/crates/ty_completion_eval/truth/auto-import-includes-modules/pyproject.toml b/crates/ty_completion_eval/truth/auto-import-includes-modules/pyproject.toml new file mode 100644 index 0000000000..cd277d8097 --- /dev/null +++ b/crates/ty_completion_eval/truth/auto-import-includes-modules/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "test" +version = "0.1.0" +requires-python = ">=3.13" +dependencies = [] diff --git a/crates/ty_completion_eval/truth/auto-import-includes-modules/uv.lock b/crates/ty_completion_eval/truth/auto-import-includes-modules/uv.lock new file mode 100644 index 0000000000..a4937d10d3 --- /dev/null +++ b/crates/ty_completion_eval/truth/auto-import-includes-modules/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[[package]] +name = "test" +version = "0.1.0" +source = { virtual = "." } From a9de6b5c3e15582a235de3ab4306104aad943e90 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 4 Dec 2025 15:17:57 -0800 Subject: [PATCH 31/41] [ty] normalize typevar bounds/constraints in cycles (#21800) Fixes https://github.com/astral-sh/ty/issues/1587 ## Summary Perform cycle normalization on typevar bounds and constraints (similar to how it was already done for typevar defaults) in order to ensure convergence in cyclic cases. There might be another fix here that could avoid the cycle in many more cases, where we don't eagerly evaluate typevar bounds/constraints on explicit specialization, but just accept the given specialization and later evaluate to see whether we need to emit a diagnostic on it. But the current fix here is sufficient to solve the problem and matches the patterns we use to ensure cycle convergence elsewhere, so it seems good for now; left a TODO for the other idea. This fix is sufficient to make us not panic, but not sufficient to get the semantics fully correct; see the TODOs in the tests. I have ideas for fixing that as well, but it seems worth at least getting this in to fix the panic. ## Test Plan Test that previously panicked now does not. --------- Co-authored-by: Alex Waygood --- .../resources/mdtest/protocols.md | 22 ++++-- crates/ty_python_semantic/src/types.rs | 79 ++++++++++++++++++- .../src/types/infer/builder.rs | 8 ++ 3 files changed, 101 insertions(+), 8 deletions(-) diff --git a/crates/ty_python_semantic/resources/mdtest/protocols.md b/crates/ty_python_semantic/resources/mdtest/protocols.md index cfa4c68914..28069bd07c 100644 --- a/crates/ty_python_semantic/resources/mdtest/protocols.md +++ b/crates/ty_python_semantic/resources/mdtest/protocols.md @@ -3184,14 +3184,9 @@ from ty_extensions import reveal_protocol_interface reveal_protocol_interface(Foo) ``` -## Known panics +## Protocols generic over TypeVars bound to forward references -### Protocols generic over TypeVars bound to forward references - -This test currently panics because the `ClassLiteral::explicit_bases` query fails to converge. See -issue . - - +Protocols can have TypeVars with forward reference bounds that form cycles. ```py from typing import Any, Protocol, TypeVar @@ -3209,6 +3204,19 @@ class A2(Protocol[T2]): class B1(A1[T3], Protocol[T3]): ... class B2(A2[T4], Protocol[T4]): ... + +# TODO should just be `B2[Any]` +reveal_type(T3.__bound__) # revealed: B2[Any] | @Todo(specialized non-generic class) + +# TODO error: [invalid-type-arguments] +def f(x: B1[int]): + pass + +reveal_type(T4.__bound__) # revealed: B1[Any] + +# error: [invalid-type-arguments] +def g(x: B2[int]): + pass ``` ## TODO diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 81d056f91a..820d31c1b0 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -9612,6 +9612,7 @@ impl<'db> TypeVarInstance<'db> { } #[salsa::tracked( + cycle_fn=lazy_bound_or_constraints_cycle_recover, cycle_initial=lazy_bound_or_constraints_cycle_initial, heap_size=ruff_memory_usage::heap_size )] @@ -9636,6 +9637,7 @@ impl<'db> TypeVarInstance<'db> { } #[salsa::tracked( + cycle_fn=lazy_bound_or_constraints_cycle_recover, cycle_initial=lazy_bound_or_constraints_cycle_initial, heap_size=ruff_memory_usage::heap_size )] @@ -9730,7 +9732,23 @@ fn lazy_bound_or_constraints_cycle_initial<'db>( None } -#[allow(clippy::ref_option)] +#[expect(clippy::ref_option)] +fn lazy_bound_or_constraints_cycle_recover<'db>( + db: &'db dyn Db, + cycle: &salsa::Cycle, + previous: &Option>, + current: Option>, + _typevar: TypeVarInstance<'db>, +) -> Option> { + // Normalize the bounds/constraints to ensure cycle convergence. + match (previous, current) { + (Some(prev), Some(current)) => Some(current.cycle_normalized(db, *prev, cycle)), + (None, Some(current)) => Some(current.recursive_type_normalized(db, cycle)), + (_, None) => None, + } +} + +#[expect(clippy::ref_option)] fn lazy_default_cycle_recover<'db>( db: &'db dyn Db, cycle: &salsa::Cycle, @@ -9738,6 +9756,7 @@ fn lazy_default_cycle_recover<'db>( default: Option>, _typevar: TypeVarInstance<'db>, ) -> Option> { + // Normalize the default to ensure cycle convergence. match (previous_default, default) { (Some(prev), Some(default)) => Some(default.cycle_normalized(db, *prev, cycle)), (None, Some(default)) => Some(default.recursive_type_normalized(db, cycle)), @@ -10106,6 +10125,64 @@ impl<'db> TypeVarBoundOrConstraints<'db> { } } + /// Normalize for cycle recovery by combining with the previous value and + /// removing divergent types introduced by the cycle. + /// + /// See [`Type::cycle_normalized`] for more details on how this works. + fn cycle_normalized(self, db: &'db dyn Db, previous: Self, cycle: &salsa::Cycle) -> Self { + match (self, previous) { + ( + TypeVarBoundOrConstraints::UpperBound(bound), + TypeVarBoundOrConstraints::UpperBound(prev_bound), + ) => { + TypeVarBoundOrConstraints::UpperBound(bound.cycle_normalized(db, prev_bound, cycle)) + } + ( + TypeVarBoundOrConstraints::Constraints(constraints), + TypeVarBoundOrConstraints::Constraints(prev_constraints), + ) => { + // Normalize each constraint with its corresponding previous constraint + let current_elements = constraints.elements(db); + let prev_elements = prev_constraints.elements(db); + TypeVarBoundOrConstraints::Constraints(UnionType::new( + db, + current_elements + .iter() + .zip(prev_elements.iter()) + .map(|(ty, prev_ty)| ty.cycle_normalized(db, *prev_ty, cycle)) + .collect::>(), + )) + } + // The choice of whether it's an upper bound or constraints is purely syntactic and + // thus can never change in a cycle: `parsed_module` does not participate in cycles, + // the AST will never change from one iteration to the next. + _ => unreachable!( + "TypeVar switched from bound to constraints (or vice versa) in fixpoint iteration" + ), + } + } + + /// Normalize recursive types for cycle recovery when there's no previous value. + /// + /// See [`Type::recursive_type_normalized`] for more details. + fn recursive_type_normalized(self, db: &'db dyn Db, cycle: &salsa::Cycle) -> Self { + match self { + TypeVarBoundOrConstraints::UpperBound(bound) => { + TypeVarBoundOrConstraints::UpperBound(bound.recursive_type_normalized(db, cycle)) + } + TypeVarBoundOrConstraints::Constraints(constraints) => { + TypeVarBoundOrConstraints::Constraints(UnionType::new( + db, + constraints + .elements(db) + .iter() + .map(|ty| ty.recursive_type_normalized(db, cycle)) + .collect::>(), + )) + } + } + } + fn materialize_impl( self, db: &'db dyn Db, diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 9488d2270d..dd04b6d001 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -11481,6 +11481,10 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { if typevar.default_type(db).is_some() { typevar_with_defaults += 1; } + // TODO consider just accepting the given specialization without checking + // against bounds/constraints, but recording the expression for deferred + // checking at end of scope. This would avoid a lot of cycles caused by eagerly + // doing assignment checks here. match typevar.typevar(db).bound_or_constraints(db) { Some(TypeVarBoundOrConstraints::UpperBound(bound)) => { if provided_type @@ -11505,6 +11509,10 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } Some(TypeVarBoundOrConstraints::Constraints(constraints)) => { + // TODO: this is wrong, the given specialization needs to be assignable + // to _at least one_ of the individual constraints, not to the union of + // all of them. `int | str` is not a valid specialization of a typevar + // constrained to `(int, str)`. if provided_type .when_assignable_to( db, From f3e5713d90f0b1bfcba9bfe3a3ab0c3df33ece83 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama <45118249+mtshiba@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:01:48 +0900 Subject: [PATCH 32/41] [ty] increase the limit on the number of elements in a non-recursively defined literal union (#21683) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Closes https://github.com/astral-sh/ty/issues/957 As explained in https://github.com/astral-sh/ty/issues/957, literal union types for recursively defined values ​​can be widened early to speed up the convergence of fixed-point iterations. This PR achieves this by embedding a marker in `UnionType` that distinguishes whether a value is recursively defined. This also allows us to identify values ​​that are not recursively defined, so I've increased the limit on the number of elements in a literal union type for such values. Edit: while this PR doesn't provide the significant performance improvement initially hoped for, it does have the benefit of allowing the number of elements in a literal union to be raised above the salsa limit, and indeed mypy_primer results revealed that a literal union of 220 elements was actually being used. ## Test Plan `call/union.md` has been updated --- .../resources/mdtest/call/union.md | 13 ++- crates/ty_python_semantic/src/types.rs | 45 ++++++++-- .../ty_python_semantic/src/types/builder.rs | 89 +++++++++++++++++-- .../src/types/infer/builder.rs | 2 + .../src/types/subclass_of.rs | 8 +- crates/ty_python_semantic/src/types/tuple.rs | 3 +- 6 files changed, 138 insertions(+), 22 deletions(-) diff --git a/crates/ty_python_semantic/resources/mdtest/call/union.md b/crates/ty_python_semantic/resources/mdtest/call/union.md index 4f374ac754..8d722288e4 100644 --- a/crates/ty_python_semantic/resources/mdtest/call/union.md +++ b/crates/ty_python_semantic/resources/mdtest/call/union.md @@ -227,17 +227,22 @@ def _(literals_2: Literal[0, 1], b: bool, flag: bool): literals_16 = 4 * literals_4 + literals_4 # Literal[0, 1, .., 15] literals_64 = 4 * literals_16 + literals_4 # Literal[0, 1, .., 63] literals_128 = 2 * literals_64 + literals_2 # Literal[0, 1, .., 127] + literals_256 = 2 * literals_128 + literals_2 # Literal[0, 1, .., 255] - # Going beyond the MAX_UNION_LITERALS limit (currently 200): - literals_256 = 16 * literals_16 + literals_16 - reveal_type(literals_256) # revealed: int + # Going beyond the MAX_UNION_LITERALS limit (currently 512): + literals_512 = 2 * literals_256 + literals_2 # Literal[0, 1, .., 511] + reveal_type(literals_512 if flag else 512) # revealed: int # Going beyond the limit when another type is already part of the union bool_and_literals_128 = b if flag else literals_128 # bool | Literal[0, 1, ..., 127] literals_128_shifted = literals_128 + 128 # Literal[128, 129, ..., 255] + literals_256_shifted = literals_256 + 256 # Literal[256, 257, ..., 511] # Now union the two: - reveal_type(bool_and_literals_128 if flag else literals_128_shifted) # revealed: int + two = bool_and_literals_128 if flag else literals_128_shifted + # revealed: bool | Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255] + reveal_type(two) + reveal_type(two if flag else literals_256_shifted) # revealed: int ``` ## Simplifying gradually-equivalent types diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 820d31c1b0..e297ebf821 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -44,6 +44,7 @@ use crate::semantic_index::scope::ScopeId; use crate::semantic_index::{imported_modules, place_table, semantic_index}; use crate::suppression::check_suppressions; use crate::types::bound_super::BoundSuperType; +use crate::types::builder::RecursivelyDefined; use crate::types::call::{Binding, Bindings, CallArguments, CallableBinding}; pub(crate) use crate::types::class_base::ClassBase; use crate::types::constraints::{ @@ -9668,6 +9669,7 @@ impl<'db> TypeVarInstance<'db> { .skip(1) .map(|arg| definition_expression_type(db, definition, arg)) .collect::>(), + RecursivelyDefined::No, ) } _ => return None, @@ -10120,6 +10122,7 @@ impl<'db> TypeVarBoundOrConstraints<'db> { .iter() .map(|ty| ty.normalized_impl(db, visitor)) .collect::>(), + constraints.recursively_defined(db), )) } } @@ -10201,6 +10204,7 @@ impl<'db> TypeVarBoundOrConstraints<'db> { .iter() .map(|ty| ty.materialize(db, materialization_kind, visitor)) .collect::>(), + RecursivelyDefined::No, )) } } @@ -13142,6 +13146,9 @@ pub struct UnionType<'db> { /// The union type includes values in any of these types. #[returns(deref)] pub elements: Box<[Type<'db>]>, + /// Whether the value pointed to by this type is recursively defined. + /// If `Yes`, union literal widening is performed early. + recursively_defined: RecursivelyDefined, } pub(crate) fn walk_union<'db, V: visitor::TypeVisitor<'db> + ?Sized>( @@ -13226,7 +13233,14 @@ impl<'db> UnionType<'db> { db: &'db dyn Db, transform_fn: impl FnMut(&Type<'db>) -> Type<'db>, ) -> Type<'db> { - Self::from_elements(db, self.elements(db).iter().map(transform_fn)) + self.elements(db) + .iter() + .map(transform_fn) + .fold(UnionBuilder::new(db), |builder, element| { + builder.add(element) + }) + .recursively_defined(self.recursively_defined(db)) + .build() } /// A fallible version of [`UnionType::map`]. @@ -13241,7 +13255,12 @@ impl<'db> UnionType<'db> { db: &'db dyn Db, transform_fn: impl FnMut(&Type<'db>) -> Option>, ) -> Option> { - Self::try_from_elements(db, self.elements(db).iter().map(transform_fn)) + let mut builder = UnionBuilder::new(db); + for element in self.elements(db).iter().map(transform_fn) { + builder = builder.add(element?); + } + builder = builder.recursively_defined(self.recursively_defined(db)); + Some(builder.build()) } pub(crate) fn to_instance(self, db: &'db dyn Db) -> Option> { @@ -13253,7 +13272,14 @@ impl<'db> UnionType<'db> { db: &'db dyn Db, mut f: impl FnMut(&Type<'db>) -> bool, ) -> Type<'db> { - Self::from_elements(db, self.elements(db).iter().filter(|ty| f(ty))) + self.elements(db) + .iter() + .filter(|ty| f(ty)) + .fold(UnionBuilder::new(db), |builder, element| { + builder.add(*element) + }) + .recursively_defined(self.recursively_defined(db)) + .build() } pub(crate) fn map_with_boundness( @@ -13288,7 +13314,9 @@ impl<'db> UnionType<'db> { Place::Undefined } else { Place::Defined( - builder.build(), + builder + .recursively_defined(self.recursively_defined(db)) + .build(), origin, if possibly_unbound { Definedness::PossiblyUndefined @@ -13336,7 +13364,9 @@ impl<'db> UnionType<'db> { Place::Undefined } else { Place::Defined( - builder.build(), + builder + .recursively_defined(self.recursively_defined(db)) + .build(), origin, if possibly_unbound { Definedness::PossiblyUndefined @@ -13371,6 +13401,7 @@ impl<'db> UnionType<'db> { .unpack_aliases(true), UnionBuilder::add, ) + .recursively_defined(self.recursively_defined(db)) .build() } @@ -13383,7 +13414,8 @@ impl<'db> UnionType<'db> { let mut builder = UnionBuilder::new(db) .order_elements(false) .unpack_aliases(false) - .cycle_recovery(true); + .cycle_recovery(true) + .recursively_defined(self.recursively_defined(db)); let mut empty = true; for ty in self.elements(db) { if nested { @@ -13398,6 +13430,7 @@ impl<'db> UnionType<'db> { // `Divergent` in a union type does not mean true divergence, so we skip it if not nested. // e.g. T | Divergent == T | (T | (T | (T | ...))) == T if ty == &div { + builder = builder.recursively_defined(RecursivelyDefined::Yes); continue; } builder = builder.add( diff --git a/crates/ty_python_semantic/src/types/builder.rs b/crates/ty_python_semantic/src/types/builder.rs index 0618682837..64ca36010a 100644 --- a/crates/ty_python_semantic/src/types/builder.rs +++ b/crates/ty_python_semantic/src/types/builder.rs @@ -202,12 +202,30 @@ enum ReduceResult<'db> { Type(Type<'db>), } -// TODO increase this once we extend `UnionElement` throughout all union/intersection -// representations, so that we can make large unions of literals fast in all operations. -// -// For now (until we solve https://github.com/astral-sh/ty/issues/957), keep this number -// below 200, which is the salsa fixpoint iteration limit. -const MAX_UNION_LITERALS: usize = 190; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, get_size2::GetSize)] +pub enum RecursivelyDefined { + Yes, + No, +} + +impl RecursivelyDefined { + const fn is_yes(self) -> bool { + matches!(self, RecursivelyDefined::Yes) + } + + const fn or(self, other: RecursivelyDefined) -> RecursivelyDefined { + match (self, other) { + (RecursivelyDefined::Yes, _) | (_, RecursivelyDefined::Yes) => RecursivelyDefined::Yes, + _ => RecursivelyDefined::No, + } + } +} + +/// If the value ​​is defined recursively, widening is performed from fewer literal elements, resulting in faster convergence of the fixed-point iteration. +const MAX_RECURSIVE_UNION_LITERALS: usize = 10; +/// If the value ​​is defined non-recursively, the fixed-point iteration will converge in one go, +/// so in principle we can have as many literal elements as we want, but to avoid unintended huge computational loads, we limit it to 256. +const MAX_NON_RECURSIVE_UNION_LITERALS: usize = 256; pub(crate) struct UnionBuilder<'db> { elements: Vec>, @@ -217,6 +235,7 @@ pub(crate) struct UnionBuilder<'db> { // This is enabled when joining types in a `cycle_recovery` function. // Since a cycle cannot be created within a `cycle_recovery` function, execution of `is_redundant_with` is skipped. cycle_recovery: bool, + recursively_defined: RecursivelyDefined, } impl<'db> UnionBuilder<'db> { @@ -227,6 +246,7 @@ impl<'db> UnionBuilder<'db> { unpack_aliases: true, order_elements: false, cycle_recovery: false, + recursively_defined: RecursivelyDefined::No, } } @@ -248,6 +268,11 @@ impl<'db> UnionBuilder<'db> { self } + pub(crate) fn recursively_defined(mut self, val: RecursivelyDefined) -> Self { + self.recursively_defined = val; + self + } + pub(crate) fn is_empty(&self) -> bool { self.elements.is_empty() } @@ -258,6 +283,27 @@ impl<'db> UnionBuilder<'db> { self.elements.push(UnionElement::Type(Type::object())); } + fn widen_literal_types(&mut self, seen_aliases: &mut Vec>) { + let mut replace_with = vec![]; + for elem in &self.elements { + match elem { + UnionElement::IntLiterals(_) => { + replace_with.push(KnownClass::Int.to_instance(self.db)); + } + UnionElement::StringLiterals(_) => { + replace_with.push(KnownClass::Str.to_instance(self.db)); + } + UnionElement::BytesLiterals(_) => { + replace_with.push(KnownClass::Bytes.to_instance(self.db)); + } + UnionElement::Type(_) => {} + } + } + for ty in replace_with { + self.add_in_place_impl(ty, seen_aliases); + } + } + /// Adds a type to this union. pub(crate) fn add(mut self, ty: Type<'db>) -> Self { self.add_in_place(ty); @@ -270,6 +316,15 @@ impl<'db> UnionBuilder<'db> { } pub(crate) fn add_in_place_impl(&mut self, ty: Type<'db>, seen_aliases: &mut Vec>) { + let cycle_recovery = self.cycle_recovery; + let should_widen = |literals, recursively_defined: RecursivelyDefined| { + if recursively_defined.is_yes() && cycle_recovery { + literals >= MAX_RECURSIVE_UNION_LITERALS + } else { + literals >= MAX_NON_RECURSIVE_UNION_LITERALS + } + }; + match ty { Type::Union(union) => { let new_elements = union.elements(self.db); @@ -277,6 +332,20 @@ impl<'db> UnionBuilder<'db> { for element in new_elements { self.add_in_place_impl(*element, seen_aliases); } + self.recursively_defined = self + .recursively_defined + .or(union.recursively_defined(self.db)); + if self.cycle_recovery && self.recursively_defined.is_yes() { + let literals = self.elements.iter().fold(0, |acc, elem| match elem { + UnionElement::IntLiterals(literals) => acc + literals.len(), + UnionElement::StringLiterals(literals) => acc + literals.len(), + UnionElement::BytesLiterals(literals) => acc + literals.len(), + UnionElement::Type(_) => acc, + }); + if should_widen(literals, self.recursively_defined) { + self.widen_literal_types(seen_aliases); + } + } } // Adding `Never` to a union is a no-op. Type::Never => {} @@ -300,7 +369,7 @@ impl<'db> UnionBuilder<'db> { for (index, element) in self.elements.iter_mut().enumerate() { match element { UnionElement::StringLiterals(literals) => { - if literals.len() >= MAX_UNION_LITERALS { + if should_widen(literals.len(), self.recursively_defined) { let replace_with = KnownClass::Str.to_instance(self.db); self.add_in_place_impl(replace_with, seen_aliases); return; @@ -345,7 +414,7 @@ impl<'db> UnionBuilder<'db> { for (index, element) in self.elements.iter_mut().enumerate() { match element { UnionElement::BytesLiterals(literals) => { - if literals.len() >= MAX_UNION_LITERALS { + if should_widen(literals.len(), self.recursively_defined) { let replace_with = KnownClass::Bytes.to_instance(self.db); self.add_in_place_impl(replace_with, seen_aliases); return; @@ -390,7 +459,7 @@ impl<'db> UnionBuilder<'db> { for (index, element) in self.elements.iter_mut().enumerate() { match element { UnionElement::IntLiterals(literals) => { - if literals.len() >= MAX_UNION_LITERALS { + if should_widen(literals.len(), self.recursively_defined) { let replace_with = KnownClass::Int.to_instance(self.db); self.add_in_place_impl(replace_with, seen_aliases); return; @@ -585,6 +654,7 @@ impl<'db> UnionBuilder<'db> { _ => Some(Type::Union(UnionType::new( self.db, types.into_boxed_slice(), + self.recursively_defined, ))), } } @@ -696,6 +766,7 @@ impl<'db> IntersectionBuilder<'db> { enum_member_literals(db, instance.class_literal(db), None) .expect("Calling `enum_member_literals` on an enum class") .collect::>(), + RecursivelyDefined::No, )), seen_aliases, ) diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index dd04b6d001..ccb25b66a2 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -50,6 +50,7 @@ use crate::semantic_index::{ ApplicableConstraints, EnclosingSnapshotResult, SemanticIndex, place_table, }; use crate::subscript::{PyIndex, PySlice}; +use crate::types::builder::RecursivelyDefined; use crate::types::call::bind::{CallableDescription, MatchingOverloadIndex}; use crate::types::call::{Binding, Bindings, CallArguments, CallError, CallErrorKind}; use crate::types::class::{CodeGeneratorKind, FieldKind, MetaclassErrorKind, MethodDecorator}; @@ -3283,6 +3284,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { elts.iter() .map(|expr| self.infer_type_expression(expr)) .collect::>(), + RecursivelyDefined::No, )); self.store_expression_type(expr, ty); } diff --git a/crates/ty_python_semantic/src/types/subclass_of.rs b/crates/ty_python_semantic/src/types/subclass_of.rs index d906a472c3..4d15700163 100644 --- a/crates/ty_python_semantic/src/types/subclass_of.rs +++ b/crates/ty_python_semantic/src/types/subclass_of.rs @@ -416,7 +416,7 @@ impl<'db> SubclassOfInner<'db> { ) } Some(TypeVarBoundOrConstraints::Constraints(constraints)) => { - let constraints = constraints + let constraints_types = constraints .elements(db) .iter() .map(|constraint| { @@ -425,7 +425,11 @@ impl<'db> SubclassOfInner<'db> { }) .collect::>(); - TypeVarBoundOrConstraints::Constraints(UnionType::new(db, constraints)) + TypeVarBoundOrConstraints::Constraints(UnionType::new( + db, + constraints_types, + constraints.recursively_defined(db), + )) } }) }); diff --git a/crates/ty_python_semantic/src/types/tuple.rs b/crates/ty_python_semantic/src/types/tuple.rs index d2b96f2849..6405ae3ae7 100644 --- a/crates/ty_python_semantic/src/types/tuple.rs +++ b/crates/ty_python_semantic/src/types/tuple.rs @@ -23,6 +23,7 @@ use itertools::{Either, EitherOrBoth, Itertools}; use crate::semantic_index::definition::Definition; use crate::subscript::{Nth, OutOfBoundsError, PyIndex, PySlice, StepSizeZeroError}; +use crate::types::builder::RecursivelyDefined; use crate::types::class::{ClassType, KnownClass}; use crate::types::constraints::{ConstraintSet, IteratorConstraintsExtension}; use crate::types::generics::InferableTypeVars; @@ -1458,7 +1459,7 @@ impl<'db> Tuple> { // those techniques ensure that union elements are deduplicated and unions are eagerly simplified // into other types where necessary. Here, however, we know that there are no duplicates // in this union, so it's probably more efficient to use `UnionType::new()` directly. - Type::Union(UnionType::new(db, elements)) + Type::Union(UnionType::new(db, elements, RecursivelyDefined::No)) }; TupleSpec::heterogeneous([ From 3511b7a06bafffdf07180f07282e799d8e7e6c02 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama <45118249+mtshiba@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:05:41 +0900 Subject: [PATCH 33/41] [ty] do nothing with `store_expression_type` if `inner_expression_inference_state` is `Get` (#21718) ## Summary Fixes https://github.com/astral-sh/ty/issues/1688 ## Test Plan N/A --- .../resources/corpus/inner_expression_inference_state.py | 6 ++++++ crates/ty_python_semantic/src/types/infer/builder.rs | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 crates/ty_python_semantic/resources/corpus/inner_expression_inference_state.py diff --git a/crates/ty_python_semantic/resources/corpus/inner_expression_inference_state.py b/crates/ty_python_semantic/resources/corpus/inner_expression_inference_state.py new file mode 100644 index 0000000000..dcf4bd462b --- /dev/null +++ b/crates/ty_python_semantic/resources/corpus/inner_expression_inference_state.py @@ -0,0 +1,6 @@ +# This is a regression test for `store_expression_type`. +# ref: https://github.com/astral-sh/ty/issues/1688 + +x: int + +type x[T] = x[T, U] diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index ccb25b66a2..d308553801 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -7108,10 +7108,13 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { #[track_caller] fn store_expression_type(&mut self, expression: &ast::Expr, ty: Type<'db>) { - if self.deferred_state.in_string_annotation() { + if self.deferred_state.in_string_annotation() + || self.inner_expression_inference_state.is_get() + { // Avoid storing the type of expressions that are part of a string annotation because // the expression ids don't exists in the semantic index. Instead, we'll store the type // on the string expression itself that represents the annotation. + // Also, if `inner_expression_inference_state` is `Get`, the expression type has already been stored. return; } From 10de34299105e3834ed965adda74ce3a26006d23 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama <45118249+mtshiba@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:20:24 +0900 Subject: [PATCH 34/41] [ty] fix build failure caused by conflicts between #21683 and #21800 (#21802) --- crates/ty_python_semantic/src/types.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index e297ebf821..8beb1cf68a 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -10154,6 +10154,7 @@ impl<'db> TypeVarBoundOrConstraints<'db> { .zip(prev_elements.iter()) .map(|(ty, prev_ty)| ty.cycle_normalized(db, *prev_ty, cycle)) .collect::>(), + constraints.recursively_defined(db), )) } // The choice of whether it's an upper bound or constraints is purely syntactic and @@ -10181,6 +10182,7 @@ impl<'db> TypeVarBoundOrConstraints<'db> { .iter() .map(|ty| ty.recursive_type_normalized(db, cycle)) .collect::>(), + constraints.recursively_defined(db), )) } } From 1951f1bbb821c991657d48381880742bd73758f2 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama <45118249+mtshiba@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:48:38 +0900 Subject: [PATCH 35/41] [ty] fix panic when instantiating a type variable with invalid constraints (#21663) --- .../corpus/invalid_typevar_constraints.py | 6 + crates/ty_python_semantic/src/types.rs | 206 +++++++++++++----- .../src/types/bound_super.rs | 4 +- .../ty_python_semantic/src/types/builder.rs | 2 +- .../src/types/infer/builder.rs | 16 +- crates/ty_python_semantic/src/types/narrow.rs | 2 +- .../src/types/subclass_of.rs | 26 +-- crates/ty_python_semantic/src/types/tuple.rs | 4 + 8 files changed, 185 insertions(+), 81 deletions(-) create mode 100644 crates/ty_python_semantic/resources/corpus/invalid_typevar_constraints.py diff --git a/crates/ty_python_semantic/resources/corpus/invalid_typevar_constraints.py b/crates/ty_python_semantic/resources/corpus/invalid_typevar_constraints.py new file mode 100644 index 0000000000..14a79363e6 --- /dev/null +++ b/crates/ty_python_semantic/resources/corpus/invalid_typevar_constraints.py @@ -0,0 +1,6 @@ +class C[T: (A, B)]: + def f(foo: T): + try: + pass + except foo: + pass diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 8beb1cf68a..61cc160769 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -68,7 +68,7 @@ pub(crate) use crate::types::narrow::infer_narrowing_constraint; use crate::types::newtype::NewType; pub(crate) use crate::types::signatures::{Parameter, Parameters}; use crate::types::signatures::{ParameterForm, walk_signature}; -use crate::types::tuple::{TupleSpec, TupleSpecBuilder}; +use crate::types::tuple::{Tuple, TupleSpec, TupleSpecBuilder}; pub(crate) use crate::types::typed_dict::{TypedDictParams, TypedDictType, walk_typed_dict_type}; pub use crate::types::variance::TypeVarVariance; use crate::types::variance::VarianceInferable; @@ -5401,9 +5401,9 @@ impl<'db> Type<'db> { Some(TypeVarBoundOrConstraints::UpperBound(bound)) => { bound.try_bool_impl(db, allow_short_circuit, visitor)? } - Some(TypeVarBoundOrConstraints::Constraints(constraints)) => { - try_union(constraints)? - } + Some(TypeVarBoundOrConstraints::Constraints(constraints)) => constraints + .as_type(db) + .try_bool_impl(db, allow_short_circuit, visitor)?, } } @@ -6453,7 +6453,7 @@ impl<'db> Type<'db> { TypeVarBoundOrConstraints::UpperBound(bound) => { non_async_special_case(db, bound) } - TypeVarBoundOrConstraints::Constraints(union) => non_async_special_case(db, Type::Union(union)), + TypeVarBoundOrConstraints::Constraints(constraints) => non_async_special_case(db, constraints.as_type(db)), }, Type::Union(union) => { let elements = union.elements(db); @@ -9594,7 +9594,7 @@ impl<'db> TypeVarInstance<'db> { TypeVarBoundOrConstraints::UpperBound(upper_bound.to_instance(db)?) } TypeVarBoundOrConstraints::Constraints(constraints) => { - TypeVarBoundOrConstraints::Constraints(constraints.to_instance(db)?.as_union()?) + TypeVarBoundOrConstraints::Constraints(constraints.to_instance(db)?) } }; let identity = TypeVarIdentity::new( @@ -9645,22 +9645,30 @@ impl<'db> TypeVarInstance<'db> { fn lazy_constraints(self, db: &'db dyn Db) -> Option> { let definition = self.definition(db)?; let module = parsed_module(db, definition.file(db)).load(db); - let ty = match definition.kind(db) { + let constraints = match definition.kind(db) { // PEP 695 typevar DefinitionKind::TypeVar(typevar) => { let typevar_node = typevar.node(&module); - definition_expression_type(db, definition, typevar_node.bound.as_ref()?) - .as_union()? + let bound = + definition_expression_type(db, definition, typevar_node.bound.as_ref()?); + let constraints = if let Some(tuple) = bound + .as_nominal_instance() + .and_then(|instance| instance.tuple_spec(db)) + { + if let Tuple::Fixed(tuple) = tuple.into_owned() { + tuple.owned_elements() + } else { + vec![Type::unknown()].into_boxed_slice() + } + } else { + vec![Type::unknown()].into_boxed_slice() + }; + TypeVarConstraints::new(db, constraints) } // legacy typevar DefinitionKind::Assignment(assignment) => { let call_expr = assignment.value(&module).as_call_expr()?; - // We don't use `UnionType::from_elements` or `UnionBuilder` here, - // because we don't want to simplify the list of constraints as we would with - // an actual union type. - // TODO: We probably shouldn't use `UnionType` to store these at all? TypeVar - // constraints are not a union. - UnionType::new( + TypeVarConstraints::new( db, call_expr .arguments @@ -9669,12 +9677,11 @@ impl<'db> TypeVarInstance<'db> { .skip(1) .map(|arg| definition_expression_type(db, definition, arg)) .collect::>(), - RecursivelyDefined::No, ) } _ => return None, }; - Some(TypeVarBoundOrConstraints::Constraints(ty)) + Some(TypeVarBoundOrConstraints::Constraints(constraints)) } #[salsa::tracked(cycle_fn=lazy_default_cycle_recover, cycle_initial=lazy_default_cycle_initial, heap_size=ruff_memory_usage::heap_size)] @@ -10086,10 +10093,133 @@ impl<'db> From> for TypeVarBoundOrConstraintsEval } } +/// Type variable constraints (e.g. `T: (int, str)`). +/// This is structurally identical to [`UnionType`], except that it does not perform simplification and preserves the element types. +#[salsa::interned(debug, heap_size=ruff_memory_usage::heap_size)] +pub struct TypeVarConstraints<'db> { + #[returns(ref)] + elements: Box<[Type<'db>]>, +} + +impl get_size2::GetSize for TypeVarConstraints<'_> {} + +fn walk_type_var_constraints<'db, V: visitor::TypeVisitor<'db> + ?Sized>( + db: &'db dyn Db, + constraints: TypeVarConstraints<'db>, + visitor: &V, +) { + for ty in constraints.elements(db) { + visitor.visit_type(db, *ty); + } +} + +impl<'db> TypeVarConstraints<'db> { + fn as_type(self, db: &'db dyn Db) -> Type<'db> { + let mut builder = UnionBuilder::new(db); + for ty in self.elements(db) { + builder = builder.add(*ty); + } + builder.build() + } + + fn to_instance(self, db: &'db dyn Db) -> Option> { + let mut instance_elements = Vec::new(); + for ty in self.elements(db) { + instance_elements.push(ty.to_instance(db)?); + } + Some(TypeVarConstraints::new( + db, + instance_elements.into_boxed_slice(), + )) + } + + fn map(self, db: &'db dyn Db, transform_fn: impl FnMut(&Type<'db>) -> Type<'db>) -> Self { + let mapped = self + .elements(db) + .iter() + .map(transform_fn) + .collect::>(); + TypeVarConstraints::new(db, mapped) + } + + pub(crate) fn map_with_boundness_and_qualifiers( + self, + db: &'db dyn Db, + mut transform_fn: impl FnMut(&Type<'db>) -> PlaceAndQualifiers<'db>, + ) -> PlaceAndQualifiers<'db> { + let mut builder = UnionBuilder::new(db); + let mut qualifiers = TypeQualifiers::empty(); + + let mut all_unbound = true; + let mut possibly_unbound = false; + let mut origin = TypeOrigin::Declared; + for ty in self.elements(db) { + let PlaceAndQualifiers { + place: ty_member, + qualifiers: new_qualifiers, + } = transform_fn(ty); + qualifiers |= new_qualifiers; + match ty_member { + Place::Undefined => { + possibly_unbound = true; + } + Place::Defined(ty_member, member_origin, member_boundness) => { + origin = origin.merge(member_origin); + if member_boundness == Definedness::PossiblyUndefined { + possibly_unbound = true; + } + + all_unbound = false; + builder = builder.add(ty_member); + } + } + } + PlaceAndQualifiers { + place: if all_unbound { + Place::Undefined + } else { + Place::Defined( + builder.build(), + origin, + if possibly_unbound { + Definedness::PossiblyUndefined + } else { + Definedness::AlwaysDefined + }, + ) + }, + qualifiers, + } + } + + fn normalized_impl(self, db: &'db dyn Db, visitor: &NormalizedVisitor<'db>) -> Self { + let normalized = self + .elements(db) + .iter() + .map(|ty| ty.normalized_impl(db, visitor)) + .collect::>(); + TypeVarConstraints::new(db, normalized) + } + + fn materialize_impl( + self, + db: &'db dyn Db, + materialization_kind: MaterializationKind, + visitor: &ApplyTypeMappingVisitor<'db>, + ) -> Self { + let materialized = self + .elements(db) + .iter() + .map(|ty| ty.materialize(db, materialization_kind, visitor)) + .collect::>(); + TypeVarConstraints::new(db, materialized) + } +} + #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, salsa::Update, get_size2::GetSize)] pub enum TypeVarBoundOrConstraints<'db> { UpperBound(Type<'db>), - Constraints(UnionType<'db>), + Constraints(TypeVarConstraints<'db>), } fn walk_type_var_bounds<'db, V: visitor::TypeVisitor<'db> + ?Sized>( @@ -10100,7 +10230,7 @@ fn walk_type_var_bounds<'db, V: visitor::TypeVisitor<'db> + ?Sized>( match bounds { TypeVarBoundOrConstraints::UpperBound(bound) => visitor.visit_type(db, bound), TypeVarBoundOrConstraints::Constraints(constraints) => { - visitor.visit_union_type(db, constraints); + walk_type_var_constraints(db, constraints, visitor); } } } @@ -10112,18 +10242,7 @@ impl<'db> TypeVarBoundOrConstraints<'db> { TypeVarBoundOrConstraints::UpperBound(bound.normalized_impl(db, visitor)) } TypeVarBoundOrConstraints::Constraints(constraints) => { - // Constraints are a non-normalized union by design (it's not really a union at - // all, we are just using a union to store the types). Normalize the types but not - // the containing union. - TypeVarBoundOrConstraints::Constraints(UnionType::new( - db, - constraints - .elements(db) - .iter() - .map(|ty| ty.normalized_impl(db, visitor)) - .collect::>(), - constraints.recursively_defined(db), - )) + TypeVarBoundOrConstraints::Constraints(constraints.normalized_impl(db, visitor)) } } } @@ -10147,14 +10266,13 @@ impl<'db> TypeVarBoundOrConstraints<'db> { // Normalize each constraint with its corresponding previous constraint let current_elements = constraints.elements(db); let prev_elements = prev_constraints.elements(db); - TypeVarBoundOrConstraints::Constraints(UnionType::new( + TypeVarBoundOrConstraints::Constraints(TypeVarConstraints::new( db, current_elements .iter() .zip(prev_elements.iter()) .map(|(ty, prev_ty)| ty.cycle_normalized(db, *prev_ty, cycle)) .collect::>(), - constraints.recursively_defined(db), )) } // The choice of whether it's an upper bound or constraints is purely syntactic and @@ -10175,15 +10293,9 @@ impl<'db> TypeVarBoundOrConstraints<'db> { TypeVarBoundOrConstraints::UpperBound(bound.recursive_type_normalized(db, cycle)) } TypeVarBoundOrConstraints::Constraints(constraints) => { - TypeVarBoundOrConstraints::Constraints(UnionType::new( - db, - constraints - .elements(db) - .iter() - .map(|ty| ty.recursive_type_normalized(db, cycle)) - .collect::>(), - constraints.recursively_defined(db), - )) + TypeVarBoundOrConstraints::Constraints( + constraints.map(db, |ty| ty.recursive_type_normalized(db, cycle)), + ) } } } @@ -10199,14 +10311,10 @@ impl<'db> TypeVarBoundOrConstraints<'db> { bound.materialize(db, materialization_kind, visitor), ), TypeVarBoundOrConstraints::Constraints(constraints) => { - TypeVarBoundOrConstraints::Constraints(UnionType::new( + TypeVarBoundOrConstraints::Constraints(constraints.materialize_impl( db, - constraints - .elements(db) - .iter() - .map(|ty| ty.materialize(db, materialization_kind, visitor)) - .collect::>(), - RecursivelyDefined::No, + materialization_kind, + visitor, )) } } diff --git a/crates/ty_python_semantic/src/types/bound_super.rs b/crates/ty_python_semantic/src/types/bound_super.rs index c67aacb323..442ae0d0b9 100644 --- a/crates/ty_python_semantic/src/types/bound_super.rs +++ b/crates/ty_python_semantic/src/types/bound_super.rs @@ -157,7 +157,7 @@ impl<'db> BoundSuperError<'db> { .map(|c| c.display(db)) .join(", ") )); - Type::Union(constraints) + constraints.as_type(db) } None => { diagnostic.info(format_args!( @@ -374,7 +374,7 @@ impl<'db> BoundSuperType<'db> { delegate_with_error_mapped(bound, Some(type_var)) } Some(TypeVarBoundOrConstraints::Constraints(constraints)) => { - delegate_with_error_mapped(Type::Union(constraints), Some(type_var)) + delegate_with_error_mapped(constraints.as_type(db), Some(type_var)) } None => delegate_with_error_mapped(Type::object(), Some(type_var)), }; diff --git a/crates/ty_python_semantic/src/types/builder.rs b/crates/ty_python_semantic/src/types/builder.rs index 64ca36010a..36977078e9 100644 --- a/crates/ty_python_semantic/src/types/builder.rs +++ b/crates/ty_python_semantic/src/types/builder.rs @@ -1255,7 +1255,7 @@ impl<'db> InnerIntersectionBuilder<'db> { speculative = speculative.add_positive(bound); } Some(TypeVarBoundOrConstraints::Constraints(constraints)) => { - speculative = speculative.add_positive(Type::Union(constraints)); + speculative = speculative.add_positive(constraints.as_type(db)); } // TypeVars without a bound or constraint implicitly have `object` as their // upper bound, and it is always a no-op to add `object` to an intersection. diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index d308553801..b33883d234 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -50,7 +50,6 @@ use crate::semantic_index::{ ApplicableConstraints, EnclosingSnapshotResult, SemanticIndex, place_table, }; use crate::subscript::{PyIndex, PySlice}; -use crate::types::builder::RecursivelyDefined; use crate::types::call::bind::{CallableDescription, MatchingOverloadIndex}; use crate::types::call::{Binding, Bindings, CallArguments, CallError, CallErrorKind}; use crate::types::class::{CodeGeneratorKind, FieldKind, MetaclassErrorKind, MethodDecorator}; @@ -3274,19 +3273,14 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { std::mem::replace(&mut self.deferred_state, DeferredExpressionState::Deferred); match bound.as_deref() { Some(expr @ ast::Expr::Tuple(ast::ExprTuple { elts, .. })) => { - // We don't use UnionType::from_elements or UnionBuilder here, because we don't - // want to simplify the list of constraints like we do with the elements of an - // actual union type. - // TODO: Consider using a new `OneOfType` connective here instead, since that - // more accurately represents the actual semantics of typevar constraints. - let ty = Type::Union(UnionType::new( + // Here, we interpret `bound` as a heterogeneous tuple and convert it to `TypeVarConstraints` in `TypeVarInstance::lazy_constraints`. + let tuple_ty = Type::heterogeneous_tuple( self.db(), elts.iter() .map(|expr| self.infer_type_expression(expr)) .collect::>(), - RecursivelyDefined::No, - )); - self.store_expression_type(expr, ty); + ); + self.store_expression_type(expr, tuple_ty); } Some(expr) => { self.infer_type_expression(expr); @@ -11521,7 +11515,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { if provided_type .when_assignable_to( db, - Type::Union(constraints), + constraints.as_type(db), InferableTypeVars::None, ) .is_never_satisfied(db) diff --git a/crates/ty_python_semantic/src/types/narrow.rs b/crates/ty_python_semantic/src/types/narrow.rs index 984f214414..4aa8b85b6f 100644 --- a/crates/ty_python_semantic/src/types/narrow.rs +++ b/crates/ty_python_semantic/src/types/narrow.rs @@ -198,7 +198,7 @@ impl ClassInfoConstraintFunction { self.generate_constraint(db, bound) } TypeVarBoundOrConstraints::Constraints(constraints) => { - self.generate_constraint(db, Type::Union(constraints)) + self.generate_constraint(db, constraints.as_type(db)) } } } diff --git a/crates/ty_python_semantic/src/types/subclass_of.rs b/crates/ty_python_semantic/src/types/subclass_of.rs index 4d15700163..898a82e086 100644 --- a/crates/ty_python_semantic/src/types/subclass_of.rs +++ b/crates/ty_python_semantic/src/types/subclass_of.rs @@ -8,7 +8,7 @@ use crate::types::{ ApplyTypeMappingVisitor, BoundTypeVarInstance, ClassType, DynamicType, FindLegacyTypeVarsVisitor, HasRelationToVisitor, IsDisjointVisitor, KnownClass, MaterializationKind, MemberLookupPolicy, NormalizedVisitor, SpecialFormType, Type, TypeContext, - TypeMapping, TypeRelation, TypeVarBoundOrConstraints, UnionType, todo_type, + TypeMapping, TypeRelation, TypeVarBoundOrConstraints, todo_type, }; use crate::{Db, FxOrderSet}; @@ -190,7 +190,9 @@ impl<'db> SubclassOfType<'db> { match bound_typevar.typevar(db).bound_or_constraints(db) { None => unreachable!(), Some(TypeVarBoundOrConstraints::UpperBound(bound)) => bound, - Some(TypeVarBoundOrConstraints::Constraints(union)) => Type::Union(union), + Some(TypeVarBoundOrConstraints::Constraints(constraints)) => { + constraints.as_type(db) + } } } }; @@ -351,7 +353,7 @@ impl<'db> SubclassOfInner<'db> { .and_then(|subclass_of| subclass_of.into_class(db)) } Some(TypeVarBoundOrConstraints::Constraints(constraints)) => { - match constraints.elements(db) { + match &**constraints.elements(db) { [bound] => Self::try_from_instance(db, *bound) .and_then(|subclass_of| subclass_of.into_class(db)), _ => Some(ClassType::object(db)), @@ -416,20 +418,10 @@ impl<'db> SubclassOfInner<'db> { ) } Some(TypeVarBoundOrConstraints::Constraints(constraints)) => { - let constraints_types = constraints - .elements(db) - .iter() - .map(|constraint| { - SubclassOfType::try_from_instance(db, *constraint) - .unwrap_or(SubclassOfType::subclass_of_unknown()) - }) - .collect::>(); - - TypeVarBoundOrConstraints::Constraints(UnionType::new( - db, - constraints_types, - constraints.recursively_defined(db), - )) + TypeVarBoundOrConstraints::Constraints(constraints.map(db, |constraint| { + SubclassOfType::try_from_instance(db, *constraint) + .unwrap_or(SubclassOfType::subclass_of_unknown()) + })) } }) }); diff --git a/crates/ty_python_semantic/src/types/tuple.rs b/crates/ty_python_semantic/src/types/tuple.rs index 6405ae3ae7..787bbe0688 100644 --- a/crates/ty_python_semantic/src/types/tuple.rs +++ b/crates/ty_python_semantic/src/types/tuple.rs @@ -349,6 +349,10 @@ impl FixedLengthTuple { &self.0 } + pub(crate) fn owned_elements(self) -> Box<[T]> { + self.0 + } + pub(crate) fn elements(&self) -> impl DoubleEndedIterator + ExactSizeIterator + '_ { self.0.iter() } From 6f03afe318f3054aed2cb4667433c256de2e581d Mon Sep 17 00:00:00 2001 From: Dhruv Manilawala Date: Fri, 5 Dec 2025 12:51:40 +0530 Subject: [PATCH 36/41] Remove unused whitespaces in test cases (#21806) These aren't used in the tests themselves. There are more instances of them in other files but those require code changes so I've left them as it is. --- crates/ty_ide/src/doc_highlights.rs | 2 +- crates/ty_ide/src/docstring.rs | 20 ++++++++++---------- crates/ty_ide/src/goto_declaration.rs | 22 +++++++++++----------- crates/ty_ide/src/goto_type_definition.rs | 16 ++++++++-------- crates/ty_ide/src/hover.rs | 16 ++++++++-------- crates/ty_ide/src/inlay_hints.rs | 2 +- 6 files changed, 39 insertions(+), 39 deletions(-) diff --git a/crates/ty_ide/src/doc_highlights.rs b/crates/ty_ide/src/doc_highlights.rs index 92b7620943..c7ad2a6c17 100644 --- a/crates/ty_ide/src/doc_highlights.rs +++ b/crates/ty_ide/src/doc_highlights.rs @@ -230,7 +230,7 @@ calc = Calculator() " def test(): # Cursor on a position with no symbol - + ", ); diff --git a/crates/ty_ide/src/docstring.rs b/crates/ty_ide/src/docstring.rs index 40c94ec5cb..0755fead73 100644 --- a/crates/ty_ide/src/docstring.rs +++ b/crates/ty_ide/src/docstring.rs @@ -824,12 +824,12 @@ mod tests { Check out this great example code:: x_y = "hello" - + if len(x_y) > 4: print(x_y) else: print("too short :(") - + print("done") You love to see it. @@ -862,12 +862,12 @@ mod tests { Check out this great example code :: x_y = "hello" - + if len(x_y) > 4: print(x_y) else: print("too short :(") - + print("done") You love to see it. @@ -901,12 +901,12 @@ mod tests { :: x_y = "hello" - + if len(x_y) > 4: print(x_y) else: print("too short :(") - + print("done") You love to see it. @@ -939,12 +939,12 @@ mod tests { let docstring = r#" Check out this great example code:: x_y = "hello" - + if len(x_y) > 4: print(x_y) else: print("too short :(") - + print("done") You love to see it. "#; @@ -975,12 +975,12 @@ mod tests { Check out this great example code:: x_y = "hello" - + if len(x_y) > 4: print(x_y) else: print("too short :(") - + print("done")"#; let docstring = Docstring::new(docstring.to_owned()); diff --git a/crates/ty_ide/src/goto_declaration.rs b/crates/ty_ide/src/goto_declaration.rs index a2f147b2d3..3e455b7533 100644 --- a/crates/ty_ide/src/goto_declaration.rs +++ b/crates/ty_ide/src/goto_declaration.rs @@ -273,7 +273,7 @@ mod tests { r#" class A: x = 1 - + def method(self): def inner(): return x # Should NOT find class variable x @@ -1255,12 +1255,12 @@ x: int = 42 r#" def outer(): x = "outer_value" - + def inner(): nonlocal x x = "modified" return x # Should find the nonlocal x declaration in outer scope - + return inner "#, ); @@ -1295,12 +1295,12 @@ def outer(): r#" def outer(): xy = "outer_value" - + def inner(): nonlocal xy xy = "modified" return x # Should find the nonlocal x declaration in outer scope - + return inner "#, ); @@ -1636,7 +1636,7 @@ def function(): def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): @@ -1675,7 +1675,7 @@ def function(): def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): @@ -1713,7 +1713,7 @@ def function(): def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): @@ -1751,7 +1751,7 @@ def function(): def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): @@ -1919,7 +1919,7 @@ def function(): class C: def __init__(self): self._value = 0 - + @property def value(self): return self._value @@ -2029,7 +2029,7 @@ def function(): r#" class MyClass: ClassType = int - + def generic_method[T](self, value: ClassType) -> T: return value "#, diff --git a/crates/ty_ide/src/goto_type_definition.rs b/crates/ty_ide/src/goto_type_definition.rs index fc5aa9aded..53cc98413d 100644 --- a/crates/ty_ide/src/goto_type_definition.rs +++ b/crates/ty_ide/src/goto_type_definition.rs @@ -1111,7 +1111,7 @@ mod tests { def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): @@ -1131,7 +1131,7 @@ mod tests { def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): @@ -1151,7 +1151,7 @@ mod tests { def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): @@ -1189,7 +1189,7 @@ mod tests { def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): @@ -1398,12 +1398,12 @@ f(**kwargs) r#" def outer(): x = "outer_value" - + def inner(): nonlocal x x = "modified" return x # Should find the nonlocal x declaration in outer scope - + return inner "#, ); @@ -1438,12 +1438,12 @@ def outer(): r#" def outer(): xy = "outer_value" - + def inner(): nonlocal xy xy = "modified" return x # Should find the nonlocal x declaration in outer scope - + return inner "#, ); diff --git a/crates/ty_ide/src/hover.rs b/crates/ty_ide/src/hover.rs index affa6054e2..63b67bc864 100644 --- a/crates/ty_ide/src/hover.rs +++ b/crates/ty_ide/src/hover.rs @@ -1708,12 +1708,12 @@ def ab(a: int, *, c: int): r#" def outer(): x = "outer_value" - + def inner(): nonlocal x x = "modified" return x # Should find the nonlocal x declaration in outer scope - + return inner "#, ); @@ -1747,12 +1747,12 @@ def outer(): r#" def outer(): xy = "outer_value" - + def inner(): nonlocal xy xy = "modified" return x # Should find the nonlocal x declaration in outer scope - + return inner "#, ); @@ -1960,7 +1960,7 @@ def function(): def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): @@ -1980,7 +1980,7 @@ def function(): def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): @@ -2018,7 +2018,7 @@ def function(): def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): @@ -2057,7 +2057,7 @@ def function(): def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index c6ddf8d846..1d26d3cdd2 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -2012,7 +2012,7 @@ mod tests { def __init__(self, pos, btn): self.position: int = pos self.button: str = btn - + def my_func(event: Click): match event: case Click(x, button=ab): From 5df8a959f54b2214af77280502b4129fb97640bb Mon Sep 17 00:00:00 2001 From: mahiro <70263039+mahiro72@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:53:08 +0900 Subject: [PATCH 37/41] Update mkdocs-material to 9.7.0 (Insiders now free) (#21797) --- .github/renovate.json5 | 8 -------- .github/workflows/ci.yaml | 17 +---------------- .github/workflows/publish-docs.yml | 20 +------------------- CONTRIBUTING.md | 13 +------------ docs/requirements-insiders.txt | 8 -------- docs/requirements.txt | 2 +- mkdocs.public.yml | 6 ------ mkdocs.insiders.yml => mkdocs.yml | 0 8 files changed, 4 insertions(+), 70 deletions(-) delete mode 100644 docs/requirements-insiders.txt delete mode 100644 mkdocs.public.yml rename mkdocs.insiders.yml => mkdocs.yml (100%) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 449ea573f0..c6a8b29a76 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -75,14 +75,6 @@ matchManagers: ["cargo"], enabled: false, }, - { - // `mkdocs-material` requires a manual update to keep the version in sync - // with `mkdocs-material-insider`. - // See: https://squidfunk.github.io/mkdocs-material/insiders/upgrade/ - matchManagers: ["pip_requirements"], - matchPackageNames: ["mkdocs-material"], - enabled: false, - }, { groupName: "pre-commit dependencies", matchManagers: ["pre-commit"], diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 64e5e2163c..83b6b83bc3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -779,8 +779,6 @@ jobs: name: "mkdocs" runs-on: ubuntu-latest timeout-minutes: 10 - env: - MKDOCS_INSIDERS_SSH_KEY_EXISTS: ${{ secrets.MKDOCS_INSIDERS_SSH_KEY != '' }} steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: @@ -788,11 +786,6 @@ jobs: - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 with: save-if: ${{ github.ref == 'refs/heads/main' }} - - name: "Add SSH key" - if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS == 'true' }} - uses: webfactory/ssh-agent@a6f90b1f127823b31d4d4a8d96047790581349bd # v0.9.1 - with: - ssh-private-key: ${{ secrets.MKDOCS_INSIDERS_SSH_KEY }} - name: "Install Rust toolchain" run: rustup show - name: Install uv @@ -800,11 +793,7 @@ jobs: with: python-version: 3.13 activate-environment: true - - name: "Install Insiders dependencies" - if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS == 'true' }} - run: uv pip install -r docs/requirements-insiders.txt - name: "Install dependencies" - if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS != 'true' }} run: uv pip install -r docs/requirements.txt - name: "Update README File" run: python scripts/transform_readme.py --target mkdocs @@ -812,12 +801,8 @@ jobs: run: python scripts/generate_mkdocs.py - name: "Check docs formatting" run: python scripts/check_docs_formatted.py - - name: "Build Insiders docs" - if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS == 'true' }} - run: mkdocs build --strict -f mkdocs.insiders.yml - name: "Build docs" - if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS != 'true' }} - run: mkdocs build --strict -f mkdocs.public.yml + run: mkdocs build --strict -f mkdocs.yml check-formatter-instability-and-black-similarity: name: "formatter instabilities and black similarity" diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 000b866c48..bd72c6a766 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -20,8 +20,6 @@ on: jobs: mkdocs: runs-on: ubuntu-latest - env: - MKDOCS_INSIDERS_SSH_KEY_EXISTS: ${{ secrets.MKDOCS_INSIDERS_SSH_KEY != '' }} steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: @@ -59,23 +57,12 @@ jobs: echo "branch_name=update-docs-$branch_display_name-$timestamp" >> "$GITHUB_ENV" echo "timestamp=$timestamp" >> "$GITHUB_ENV" - - name: "Add SSH key" - if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS == 'true' }} - uses: webfactory/ssh-agent@a6f90b1f127823b31d4d4a8d96047790581349bd # v0.9.1 - with: - ssh-private-key: ${{ secrets.MKDOCS_INSIDERS_SSH_KEY }} - - name: "Install Rust toolchain" run: rustup show - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - - name: "Install Insiders dependencies" - if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS == 'true' }} - run: pip install -r docs/requirements-insiders.txt - - name: "Install dependencies" - if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS != 'true' }} run: pip install -r docs/requirements.txt - name: "Copy README File" @@ -83,13 +70,8 @@ jobs: python scripts/transform_readme.py --target mkdocs python scripts/generate_mkdocs.py - - name: "Build Insiders docs" - if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS == 'true' }} - run: mkdocs build --strict -f mkdocs.insiders.yml - - name: "Build docs" - if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS != 'true' }} - run: mkdocs build --strict -f mkdocs.public.yml + run: mkdocs build --strict -f mkdocs.yml - name: "Clone docs repo" run: git clone https://${{ secrets.ASTRAL_DOCS_PAT }}@github.com/astral-sh/docs.git astral-docs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bb6758451f..1851d45199 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -331,13 +331,6 @@ you addressed them. ## MkDocs -> [!NOTE] -> -> The documentation uses Material for MkDocs Insiders, which is closed-source software. -> This means only members of the Astral organization can preview the documentation exactly as it -> will appear in production. -> Outside contributors can still preview the documentation, but there will be some differences. Consult [the Material for MkDocs documentation](https://squidfunk.github.io/mkdocs-material/insiders/benefits/#features) for which features are exclusively available in the insiders version. - To preview any changes to the documentation locally: 1. Install the [Rust toolchain](https://www.rust-lang.org/tools/install). @@ -351,11 +344,7 @@ To preview any changes to the documentation locally: 1. Run the development server with: ```shell - # For contributors. - uvx --with-requirements docs/requirements.txt -- mkdocs serve -f mkdocs.public.yml - - # For members of the Astral org, which has access to MkDocs Insiders via sponsorship. - uvx --with-requirements docs/requirements-insiders.txt -- mkdocs serve -f mkdocs.insiders.yml + uvx --with-requirements docs/requirements.txt -- mkdocs serve -f mkdocs.yml ``` The documentation should then be available locally at diff --git a/docs/requirements-insiders.txt b/docs/requirements-insiders.txt deleted file mode 100644 index 9726b8ab5a..0000000000 --- a/docs/requirements-insiders.txt +++ /dev/null @@ -1,8 +0,0 @@ -PyYAML==6.0.3 -ruff==0.14.7 -mkdocs==1.6.1 -mkdocs-material @ git+ssh://git@github.com/astral-sh/mkdocs-material-insiders.git@39da7a5e761410349e9a1b8abf593b0cdd5453ff -mkdocs-redirects==1.2.2 -mdformat==0.7.22 -mdformat-mkdocs==4.4.2 -mkdocs-github-admonitions-plugin @ git+https://github.com/PGijsbers/admonitions.git#7343d2f4a92e4d1491094530ef3d0d02d93afbb7 diff --git a/docs/requirements.txt b/docs/requirements.txt index a9b415267f..c80ac6c61b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ PyYAML==6.0.3 ruff==0.14.7 mkdocs==1.6.1 -mkdocs-material==9.5.38 +mkdocs-material==9.7.0 mkdocs-redirects==1.2.2 mdformat==0.7.22 mdformat-mkdocs==4.4.2 diff --git a/mkdocs.public.yml b/mkdocs.public.yml deleted file mode 100644 index b6d7ae6eca..0000000000 --- a/mkdocs.public.yml +++ /dev/null @@ -1,6 +0,0 @@ -INHERIT: mkdocs.generated.yml -# Omit the `typeset` plugin which is only available in the Insiders version. -plugins: - - search -watch: - - mkdocs.generated.yml diff --git a/mkdocs.insiders.yml b/mkdocs.yml similarity index 100% rename from mkdocs.insiders.yml rename to mkdocs.yml From 3deb7e1b90daac0d9f4dbff51b031fe22eecad20 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Fri, 5 Dec 2025 11:56:28 +0100 Subject: [PATCH 38/41] [ty] Add some attribute/method renaming test cases (#21809) --- crates/ty_ide/src/rename.rs | 140 ++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/crates/ty_ide/src/rename.rs b/crates/ty_ide/src/rename.rs index 10ef9c197f..e2dcf5d93d 100644 --- a/crates/ty_ide/src/rename.rs +++ b/crates/ty_ide/src/rename.rs @@ -1426,4 +1426,144 @@ result = func(10, y=20) | "); } + + // TODO: This should rename all overloads + #[test] + fn rename_overloaded_function() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + from typing import overload, Any + + @overload + def test() -> None: ... + @overload + def test(a: str) -> str: ... + @overload + def test(a: int) -> int: ... + + def test(a: Any) -> Any: + return a + "#, + ) + .source( + "mypackage/subpkg/__init__.py", + r#" + from lib import test + + test("test") + "#, + ) + .build(); + + assert_snapshot!(test.rename("better_name"), @r" + info[rename]: Rename symbol (found 1 locations) + --> mypackage/__init__.py:5:5 + | + 4 | @overload + 5 | def test() -> None: ... + | ^^^^ + 6 | @overload + 7 | def test(a: str) -> str: ... + | + "); + } + + #[test] + fn rename_attribute() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + class Test: + attribute: str + + def __init__(self, value: str): + self.attribute = value + + class Child(Test): + def test(self): + return self.attribute + + + c = Child("test") + + print(c.attribute) + c.attribute = "new_value" + "#, + ) + .build(); + + assert_snapshot!(test.rename("better_name"), @r#" + info[rename]: Rename symbol (found 5 locations) + --> mypackage/__init__.py:3:5 + | + 2 | class Test: + 3 | attribute: str + | ^^^^^^^^^ + 4 | + 5 | def __init__(self, value: str): + 6 | self.attribute = value + | --------- + 7 | + 8 | class Child(Test): + 9 | def test(self): + 10 | return self.attribute + | --------- + | + ::: mypackage/__init__.py:15:9 + | + 13 | c = Child("test") + 14 | + 15 | print(c.attribute) + | --------- + 16 | c.attribute = "new_value" + | --------- + | + "#); + } + + // TODO: This should rename all attribute usages + // Note: Pylance only renames the assignment in `__init__`. + #[test] + fn rename_implicit_attribute() { + let test = CursorTest::builder() + .source( + "mypackage/__init__.py", + r#" + class Test: + def __init__(self, value: str): + self.attribute = value + + class Child(Test): + def __init__(self, value: str): + super().__init__(value) + self.attribute = value + "child" + + def test(self): + return self.attribute + + + c = Child("test") + + print(c.attribute) + c.attribute = "new_value" + "#, + ) + .build(); + + assert_snapshot!(test.rename("better_name"), @r" + info[rename]: Rename symbol (found 1 locations) + --> mypackage/__init__.py:4:14 + | + 2 | class Test: + 3 | def __init__(self, value: str): + 4 | self.attribute = value + | ^^^^^^^^^ + 5 | + 6 | class Child(Test): + | + "); + } } From 48f7f42784cc86763f630313746542e47f148f3a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Dec 2025 12:33:30 +0000 Subject: [PATCH 39/41] [ty] Minor improvements to `assert_type` diagnostics (#21811) --- .../mdtest/directives/assert_type.md | 3 ++- ...sert_type`_-_Basic_(c507788da2659ec9).snap | 24 ++++++++++++++++--- .../ty_python_semantic/src/types/function.rs | 23 +++++++++++------- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/crates/ty_python_semantic/resources/mdtest/directives/assert_type.md b/crates/ty_python_semantic/resources/mdtest/directives/assert_type.md index 840e8ce4b1..7f99251017 100644 --- a/crates/ty_python_semantic/resources/mdtest/directives/assert_type.md +++ b/crates/ty_python_semantic/resources/mdtest/directives/assert_type.md @@ -7,10 +7,11 @@ ```py from typing_extensions import assert_type -def _(x: int): +def _(x: int, y: bool): assert_type(x, int) # fine assert_type(x, str) # error: [type-assertion-failure] assert_type(assert_type(x, int), int) + assert_type(y, int) # error: [type-assertion-failure] ``` ## Narrowing diff --git a/crates/ty_python_semantic/resources/mdtest/snapshots/assert_type.md_-_`assert_type`_-_Basic_(c507788da2659ec9).snap b/crates/ty_python_semantic/resources/mdtest/snapshots/assert_type.md_-_`assert_type`_-_Basic_(c507788da2659ec9).snap index 42b1ad283f..e9118a57cb 100644 --- a/crates/ty_python_semantic/resources/mdtest/snapshots/assert_type.md_-_`assert_type`_-_Basic_(c507788da2659ec9).snap +++ b/crates/ty_python_semantic/resources/mdtest/snapshots/assert_type.md_-_`assert_type`_-_Basic_(c507788da2659ec9).snap @@ -14,10 +14,11 @@ mdtest path: crates/ty_python_semantic/resources/mdtest/directives/assert_type.m ``` 1 | from typing_extensions import assert_type 2 | -3 | def _(x: int): +3 | def _(x: int, y: bool): 4 | assert_type(x, int) # fine 5 | assert_type(x, str) # error: [type-assertion-failure] 6 | assert_type(assert_type(x, int), int) +7 | assert_type(y, int) # error: [type-assertion-failure] ``` # Diagnostics @@ -26,15 +27,32 @@ mdtest path: crates/ty_python_semantic/resources/mdtest/directives/assert_type.m error[type-assertion-failure]: Argument does not have asserted type `str` --> src/mdtest_snippet.py:5:5 | -3 | def _(x: int): +3 | def _(x: int, y: bool): 4 | assert_type(x, int) # fine 5 | assert_type(x, str) # error: [type-assertion-failure] | ^^^^^^^^^^^^-^^^^^^ | | - | Inferred type of argument is `int` + | Inferred type is `int` 6 | assert_type(assert_type(x, int), int) +7 | assert_type(y, int) # error: [type-assertion-failure] | info: `str` and `int` are not equivalent types info: rule `type-assertion-failure` is enabled by default ``` + +``` +error[type-assertion-failure]: Argument does not have asserted type `int` + --> src/mdtest_snippet.py:7:5 + | +5 | assert_type(x, str) # error: [type-assertion-failure] +6 | assert_type(assert_type(x, int), int) +7 | assert_type(y, int) # error: [type-assertion-failure] + | ^^^^^^^^^^^^-^^^^^^ + | | + | Inferred type is `bool` + | +info: `bool` is a subtype of `int`, but they are not equivalent +info: rule `type-assertion-failure` is enabled by default + +``` diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index f8537c27a7..0fba3aecd8 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -1483,17 +1483,22 @@ impl KnownFunction { diagnostic.annotate( Annotation::secondary(context.span(&call_expression.arguments.args[0])) - .message(format_args!( - "Inferred type of argument is `{}`", - actual_ty.display(db), - )), + .message(format_args!("Inferred type is `{}`", actual_ty.display(db),)), ); - diagnostic.info(format_args!( - "`{asserted_type}` and `{inferred_type}` are not equivalent types", - asserted_type = asserted_ty.display(db), - inferred_type = actual_ty.display(db), - )); + if actual_ty.is_subtype_of(db, *asserted_ty) { + diagnostic.info(format_args!( + "`{inferred_type}` is a subtype of `{asserted_type}`, but they are not equivalent", + asserted_type = asserted_ty.display(db), + inferred_type = actual_ty.display(db), + )); + } else { + diagnostic.info(format_args!( + "`{asserted_type}` and `{inferred_type}` are not equivalent types", + asserted_type = asserted_ty.display(db), + inferred_type = actual_ty.display(db), + )); + } diagnostic.set_concise_message(format_args!( "Type `{}` does not match asserted type `{}`", From 71a7a03ad4f0b7d2de0413abd80c1965383c0225 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Dec 2025 12:41:31 +0000 Subject: [PATCH 40/41] [ty] Add more tests for renamings (#21810) --- crates/ty_ide/src/rename.rs | 486 +++++++++++++++++++++++++++++++++++- 1 file changed, 477 insertions(+), 9 deletions(-) diff --git a/crates/ty_ide/src/rename.rs b/crates/ty_ide/src/rename.rs index e2dcf5d93d..8cb5910c60 100644 --- a/crates/ty_ide/src/rename.rs +++ b/crates/ty_ide/src/rename.rs @@ -1432,7 +1432,7 @@ result = func(10, y=20) fn rename_overloaded_function() { let test = CursorTest::builder() .source( - "mypackage/__init__.py", + "lib1.py", r#" from typing import overload, Any @@ -1448,9 +1448,9 @@ result = func(10, y=20) "#, ) .source( - "mypackage/subpkg/__init__.py", + "main.py", r#" - from lib import test + from lib2 import test test("test") "#, @@ -1459,7 +1459,7 @@ result = func(10, y=20) assert_snapshot!(test.rename("better_name"), @r" info[rename]: Rename symbol (found 1 locations) - --> mypackage/__init__.py:5:5 + --> lib1.py:5:5 | 4 | @overload 5 | def test() -> None: ... @@ -1470,11 +1470,479 @@ result = func(10, y=20) "); } + #[test] + fn rename_property() { + let test = CursorTest::builder() + .source( + "lib.py", + r#" + class Foo: + @property + def my_property(self) -> int: + return 42 + "#, + ) + .source( + "main.py", + r#" + from lib import Foo + + print(Foo().my_property) + "#, + ) + .build(); + + assert_snapshot!(test.rename("better_name"), @r" + info[rename]: Rename symbol (found 2 locations) + --> lib.py:4:9 + | + 2 | class Foo: + 3 | @property + 4 | def my_property(self) -> int: + | ^^^^^^^^^^^ + 5 | return 42 + | + ::: main.py:4:13 + | + 2 | from lib import Foo + 3 | + 4 | print(Foo().my_property) + | ----------- + | + "); + } + + // TODO: this should rename the name of the function decorated with + // `@my_property.setter` as well as the getter function name + #[test] + fn rename_property_with_setter() { + let test = CursorTest::builder() + .source( + "lib.py", + r#" + class Foo: + @property + def my_property(self) -> int: + return 42 + + @my_property.setter + def my_property(self, value: int) -> None: + pass + "#, + ) + .source( + "main.py", + r#" + from lib import Foo + + print(Foo().my_property) + Foo().my_property = 56 + "#, + ) + .build(); + + assert_snapshot!(test.rename("better_name"), @r" + info[rename]: Rename symbol (found 4 locations) + --> lib.py:4:9 + | + 2 | class Foo: + 3 | @property + 4 | def my_property(self) -> int: + | ^^^^^^^^^^^ + 5 | return 42 + 6 | + 7 | @my_property.setter + | ----------- + 8 | def my_property(self, value: int) -> None: + 9 | pass + | + ::: main.py:4:13 + | + 2 | from lib import Foo + 3 | + 4 | print(Foo().my_property) + | ----------- + 5 | Foo().my_property = 56 + | ----------- + | + "); + } + + // TODO: this should rename the name of the function decorated with + // `@my_property.deleter` as well as the getter function name + #[test] + fn rename_property_with_deleter() { + let test = CursorTest::builder() + .source( + "lib.py", + r#" + class Foo: + @property + def my_property(self) -> int: + return 42 + + @my_property.deleter + def my_property(self) -> None: + pass + "#, + ) + .source( + "main.py", + r#" + from lib import Foo + + print(Foo().my_property) + del Foo().my_property + "#, + ) + .build(); + + assert_snapshot!(test.rename("better_name"), @r" + info[rename]: Rename symbol (found 4 locations) + --> lib.py:4:9 + | + 2 | class Foo: + 3 | @property + 4 | def my_property(self) -> int: + | ^^^^^^^^^^^ + 5 | return 42 + 6 | + 7 | @my_property.deleter + | ----------- + 8 | def my_property(self) -> None: + 9 | pass + | + ::: main.py:4:13 + | + 2 | from lib import Foo + 3 | + 4 | print(Foo().my_property) + | ----------- + 5 | del Foo().my_property + | ----------- + | + "); + } + + // TODO: this should rename the name of the functions decorated with + // `@my_property.deleter` and `@my_property.deleter` as well as the + // getter function name + #[test] + fn rename_property_with_setter_and_deleter() { + let test = CursorTest::builder() + .source( + "lib.py", + r#" + class Foo: + @property + def my_property(self) -> int: + return 42 + + @my_property.setter + def my_property(self, value: int) -> None: + pass + + @my_property.deleter + def my_property(self) -> None: + pass + "#, + ) + .source( + "main.py", + r#" + from lib import Foo + + print(Foo().my_property) + Foo().my_property = 56 + del Foo().my_property + "#, + ) + .build(); + + assert_snapshot!(test.rename("better_name"), @r" + info[rename]: Rename symbol (found 6 locations) + --> lib.py:4:9 + | + 2 | class Foo: + 3 | @property + 4 | def my_property(self) -> int: + | ^^^^^^^^^^^ + 5 | return 42 + 6 | + 7 | @my_property.setter + | ----------- + 8 | def my_property(self, value: int) -> None: + 9 | pass + 10 | + 11 | @my_property.deleter + | ----------- + 12 | def my_property(self) -> None: + 13 | pass + | + ::: main.py:4:13 + | + 2 | from lib import Foo + 3 | + 4 | print(Foo().my_property) + | ----------- + 5 | Foo().my_property = 56 + | ----------- + 6 | del Foo().my_property + | ----------- + | + "); + } + + #[test] + fn rename_single_dispatch_function() { + let test = CursorTest::builder() + .source( + "foo.py", + r#" + from functools import singledispatch + + @singledispatch + def f(x: object): + raise NotImplementedError + + @f.register + def _(x: int) -> str: + return "int" + + @f.register + def _(x: str) -> int: + return int(x) + "#, + ) + .build(); + + assert_snapshot!(test.rename("better_name"), @r#" + info[rename]: Rename symbol (found 3 locations) + --> foo.py:5:5 + | + 4 | @singledispatch + 5 | def f(x: object): + | ^ + 6 | raise NotImplementedError + 7 | + 8 | @f.register + | - + 9 | def _(x: int) -> str: + 10 | return "int" + 11 | + 12 | @f.register + | - + 13 | def _(x: str) -> int: + 14 | return int(x) + | + "#); + } + + #[test] + fn rename_single_dispatch_function_stacked_register() { + let test = CursorTest::builder() + .source( + "foo.py", + r#" + from functools import singledispatch + + @singledispatch + def f(x): + raise NotImplementedError + + @f.register(int) + @f.register(float) + def _(x) -> float: + return "int" + + @f.register(str) + def _(x) -> int: + return int(x) + "#, + ) + .build(); + + assert_snapshot!(test.rename("better_name"), @r#" + info[rename]: Rename symbol (found 4 locations) + --> foo.py:5:5 + | + 4 | @singledispatch + 5 | def f(x): + | ^ + 6 | raise NotImplementedError + 7 | + 8 | @f.register(int) + | - + 9 | @f.register(float) + | - + 10 | def _(x) -> float: + 11 | return "int" + 12 | + 13 | @f.register(str) + | - + 14 | def _(x) -> int: + 15 | return int(x) + | + "#); + } + + #[test] + fn rename_single_dispatchmethod() { + let test = CursorTest::builder() + .source( + "foo.py", + r#" + from functools import singledispatchmethod + + class Foo: + @singledispatchmethod + def f(self, x: object): + raise NotImplementedError + + @f.register + def _(self, x: str) -> float: + return "int" + + @f.register + def _(self, x: str) -> int: + return int(x) + "#, + ) + .build(); + + assert_snapshot!(test.rename("better_name"), @r#" + info[rename]: Rename symbol (found 3 locations) + --> foo.py:6:9 + | + 4 | class Foo: + 5 | @singledispatchmethod + 6 | def f(self, x: object): + | ^ + 7 | raise NotImplementedError + 8 | + 9 | @f.register + | - + 10 | def _(self, x: str) -> float: + 11 | return "int" + 12 | + 13 | @f.register + | - + 14 | def _(self, x: str) -> int: + 15 | return int(x) + | + "#); + } + + #[test] + fn rename_single_dispatchmethod_staticmethod() { + let test = CursorTest::builder() + .source( + "foo.py", + r#" + from functools import singledispatchmethod + + class Foo: + @singledispatchmethod + @staticmethod + def f(self, x): + raise NotImplementedError + + @f.register(str) + @staticmethod + def _(x: int) -> str: + return "int" + + @f.register + @staticmethod + def _(x: str) -> int: + return int(x) + "#, + ) + .build(); + + assert_snapshot!(test.rename("better_name"), @r#" + info[rename]: Rename symbol (found 3 locations) + --> foo.py:7:9 + | + 5 | @singledispatchmethod + 6 | @staticmethod + 7 | def f(self, x): + | ^ + 8 | raise NotImplementedError + 9 | + 10 | @f.register(str) + | - + 11 | @staticmethod + 12 | def _(x: int) -> str: + 13 | return "int" + 14 | + 15 | @f.register + | - + 16 | @staticmethod + 17 | def _(x: str) -> int: + | + "#); + } + + #[test] + fn rename_single_dispatchmethod_classmethod() { + let test = CursorTest::builder() + .source( + "foo.py", + r#" + from functools import singledispatchmethod + + class Foo: + @singledispatchmethod + @classmethod + def f(cls, x): + raise NotImplementedError + + @f.register(str) + @classmethod + def _(cls, x) -> str: + return "int" + + @f.register(int) + @f.register(float) + @staticmethod + def _(cls, x) -> int: + return int(x) + "#, + ) + .build(); + + assert_snapshot!(test.rename("better_name"), @r#" + info[rename]: Rename symbol (found 4 locations) + --> foo.py:7:9 + | + 5 | @singledispatchmethod + 6 | @classmethod + 7 | def f(cls, x): + | ^ + 8 | raise NotImplementedError + 9 | + 10 | @f.register(str) + | - + 11 | @classmethod + 12 | def _(cls, x) -> str: + 13 | return "int" + 14 | + 15 | @f.register(int) + | - + 16 | @f.register(float) + | - + 17 | @staticmethod + 18 | def _(cls, x) -> int: + | + "#); + } + #[test] fn rename_attribute() { let test = CursorTest::builder() .source( - "mypackage/__init__.py", + "foo.py", r#" class Test: attribute: str @@ -1497,7 +1965,7 @@ result = func(10, y=20) assert_snapshot!(test.rename("better_name"), @r#" info[rename]: Rename symbol (found 5 locations) - --> mypackage/__init__.py:3:5 + --> foo.py:3:5 | 2 | class Test: 3 | attribute: str @@ -1512,7 +1980,7 @@ result = func(10, y=20) 10 | return self.attribute | --------- | - ::: mypackage/__init__.py:15:9 + ::: foo.py:15:9 | 13 | c = Child("test") 14 | @@ -1530,7 +1998,7 @@ result = func(10, y=20) fn rename_implicit_attribute() { let test = CursorTest::builder() .source( - "mypackage/__init__.py", + "main.py", r#" class Test: def __init__(self, value: str): @@ -1555,7 +2023,7 @@ result = func(10, y=20) assert_snapshot!(test.rename("better_name"), @r" info[rename]: Rename symbol (found 1 locations) - --> mypackage/__init__.py:4:14 + --> main.py:4:14 | 2 | class Test: 3 | def __init__(self, value: str): From e42cdf84957b178dc7e37aed0f2b1f4d5f0c272a Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Fri, 5 Dec 2025 08:57:21 -0500 Subject: [PATCH 41/41] [ty] Carry generic context through when converting class into `Callable` (#21798) When converting a class (whether specialized or not) into a `Callable` type, we should carry through any generic context that the constructor has. This includes both the generic context of the class itself (if it's generic) and of the constructor methods (if they are separately generic). To help test this, this also updates the `generic_context` extension function to work on `Callable` types and unions; and adds a new `into_callable` extension function that works just like `CallableTypeOf`, but on value forms instead of type forms. Pulled this out of #21551 for separate review. --- .../mdtest/generics/legacy/classes.md | 87 +++++++++++++++++ .../mdtest/generics/pep695/classes.md | 97 +++++++++++++++++++ .../resources/mdtest/liskov.md | 2 +- crates/ty_python_semantic/src/types.rs | 9 +- .../ty_python_semantic/src/types/call/bind.rs | 84 +++++++++------- crates/ty_python_semantic/src/types/class.rs | 94 ++++++++++++------ .../ty_python_semantic/src/types/function.rs | 4 + .../ty_python_semantic/src/types/generics.rs | 39 +++++++- .../src/types/signatures.rs | 7 +- .../ty_extensions/ty_extensions.pyi | 4 + 10 files changed, 350 insertions(+), 77 deletions(-) diff --git a/crates/ty_python_semantic/resources/mdtest/generics/legacy/classes.md b/crates/ty_python_semantic/resources/mdtest/generics/legacy/classes.md index 9e0696a5c2..bc6ccdb7c1 100644 --- a/crates/ty_python_semantic/resources/mdtest/generics/legacy/classes.md +++ b/crates/ty_python_semantic/resources/mdtest/generics/legacy/classes.md @@ -301,6 +301,7 @@ consistent with each other. ```py from typing_extensions import Generic, TypeVar +from ty_extensions import generic_context, into_callable T = TypeVar("T") @@ -308,6 +309,11 @@ class C(Generic[T]): def __new__(cls, x: T) -> "C[T]": return object.__new__(cls) +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C(1)) # revealed: C[int] # error: [invalid-assignment] "Object of type `C[int | str]` is not assignable to `C[int]`" @@ -318,12 +324,18 @@ wrong_innards: C[int] = C("five") ```py from typing_extensions import Generic, TypeVar +from ty_extensions import generic_context, into_callable T = TypeVar("T") class C(Generic[T]): def __init__(self, x: T) -> None: ... +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C(1)) # revealed: C[int] # error: [invalid-assignment] "Object of type `C[int | str]` is not assignable to `C[int]`" @@ -334,6 +346,7 @@ wrong_innards: C[int] = C("five") ```py from typing_extensions import Generic, TypeVar +from ty_extensions import generic_context, into_callable T = TypeVar("T") @@ -343,6 +356,11 @@ class C(Generic[T]): def __init__(self, x: T) -> None: ... +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C(1)) # revealed: C[int] # error: [invalid-assignment] "Object of type `C[int | str]` is not assignable to `C[int]`" @@ -353,6 +371,7 @@ wrong_innards: C[int] = C("five") ```py from typing_extensions import Generic, TypeVar +from ty_extensions import generic_context, into_callable T = TypeVar("T") @@ -362,6 +381,11 @@ class C(Generic[T]): def __init__(self, x: T) -> None: ... +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C(1)) # revealed: C[int] # error: [invalid-assignment] "Object of type `C[int | str]` is not assignable to `C[int]`" @@ -373,6 +397,11 @@ class D(Generic[T]): def __init__(self, *args, **kwargs) -> None: ... +# revealed: ty_extensions.GenericContext[T@D] +reveal_type(generic_context(D)) +# revealed: ty_extensions.GenericContext[T@D] +reveal_type(generic_context(into_callable(D))) + reveal_type(D(1)) # revealed: D[int] # error: [invalid-assignment] "Object of type `D[int | str]` is not assignable to `D[int]`" @@ -386,6 +415,7 @@ to specialize the class. ```py from typing_extensions import Generic, TypeVar +from ty_extensions import generic_context, into_callable T = TypeVar("T") U = TypeVar("U") @@ -398,6 +428,11 @@ class C(Generic[T, U]): class D(C[V, int]): def __init__(self, x: V) -> None: ... +# revealed: ty_extensions.GenericContext[V@D] +reveal_type(generic_context(D)) +# revealed: ty_extensions.GenericContext[V@D] +reveal_type(generic_context(into_callable(D))) + reveal_type(D(1)) # revealed: D[int] ``` @@ -405,6 +440,7 @@ reveal_type(D(1)) # revealed: D[int] ```py from typing_extensions import Generic, TypeVar +from ty_extensions import generic_context, into_callable T = TypeVar("T") U = TypeVar("U") @@ -415,6 +451,11 @@ class C(Generic[T, U]): class D(C[T, U]): pass +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(D)) +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(into_callable(D))) + reveal_type(C(1, "str")) # revealed: C[int, str] reveal_type(D(1, "str")) # revealed: D[int, str] ``` @@ -425,6 +466,7 @@ This is a specific example of the above, since it was reported specifically by a ```py from typing_extensions import Generic, TypeVar +from ty_extensions import generic_context, into_callable T = TypeVar("T") U = TypeVar("U") @@ -432,6 +474,11 @@ U = TypeVar("U") class D(dict[T, U]): pass +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(D)) +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(into_callable(D))) + reveal_type(D(key=1)) # revealed: D[str, int] ``` @@ -443,12 +490,18 @@ context. But from the user's point of view, this is another example of the above ```py from typing_extensions import Generic, TypeVar +from ty_extensions import generic_context, into_callable T = TypeVar("T") U = TypeVar("U") class C(tuple[T, U]): ... +# revealed: ty_extensions.GenericContext[T@C, U@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C, U@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C((1, 2))) # revealed: C[int, int] ``` @@ -480,6 +533,7 @@ def func8(t1: tuple[complex, list[int]], t2: tuple[int, *tuple[str, ...]], t3: t ```py from typing_extensions import Generic, TypeVar +from ty_extensions import generic_context, into_callable S = TypeVar("S") T = TypeVar("T") @@ -487,6 +541,11 @@ T = TypeVar("T") class C(Generic[T]): def __init__(self, x: T, y: S) -> None: ... +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C, S@__init__] +reveal_type(generic_context(into_callable(C))) + reveal_type(C(1, 1)) # revealed: C[int] reveal_type(C(1, "string")) # revealed: C[int] reveal_type(C(1, True)) # revealed: C[int] @@ -499,6 +558,7 @@ wrong_innards: C[int] = C("five", 1) ```py from typing_extensions import overload, Generic, TypeVar +from ty_extensions import generic_context, into_callable T = TypeVar("T") U = TypeVar("U") @@ -514,6 +574,11 @@ class C(Generic[T]): def __init__(self, x: int) -> None: ... def __init__(self, x: str | bytes | int) -> None: ... +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C("string")) # revealed: C[str] reveal_type(C(b"bytes")) # revealed: C[bytes] reveal_type(C(12)) # revealed: C[Unknown] @@ -541,6 +606,11 @@ class D(Generic[T, U]): def __init__(self, t: T, u: U) -> None: ... def __init__(self, *args) -> None: ... +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(D)) +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(into_callable(D))) + reveal_type(D("string")) # revealed: D[str, str] reveal_type(D(1)) # revealed: D[str, int] reveal_type(D(1, "string")) # revealed: D[int, str] @@ -551,6 +621,7 @@ reveal_type(D(1, "string")) # revealed: D[int, str] ```py from dataclasses import dataclass from typing_extensions import Generic, TypeVar +from ty_extensions import generic_context, into_callable T = TypeVar("T") @@ -558,6 +629,11 @@ T = TypeVar("T") class A(Generic[T]): x: T +# revealed: ty_extensions.GenericContext[T@A] +reveal_type(generic_context(A)) +# revealed: ty_extensions.GenericContext[T@A] +reveal_type(generic_context(into_callable(A))) + reveal_type(A(x=1)) # revealed: A[int] ``` @@ -565,17 +641,28 @@ reveal_type(A(x=1)) # revealed: A[int] ```py from typing_extensions import Generic, TypeVar +from ty_extensions import generic_context, into_callable T = TypeVar("T") U = TypeVar("U", default=T) class C(Generic[T, U]): ... +# revealed: ty_extensions.GenericContext[T@C, U@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C, U@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C()) # revealed: C[Unknown, Unknown] class D(Generic[T, U]): def __init__(self) -> None: ... +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(D)) +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(into_callable(D))) + reveal_type(D()) # revealed: D[Unknown, Unknown] ``` diff --git a/crates/ty_python_semantic/resources/mdtest/generics/pep695/classes.md b/crates/ty_python_semantic/resources/mdtest/generics/pep695/classes.md index a110783ed2..dbb249b45e 100644 --- a/crates/ty_python_semantic/resources/mdtest/generics/pep695/classes.md +++ b/crates/ty_python_semantic/resources/mdtest/generics/pep695/classes.md @@ -264,12 +264,19 @@ signatures don't count towards variance). ### `__new__` only ```py +from ty_extensions import generic_context, into_callable + class C[T]: x: T def __new__(cls, x: T) -> "C[T]": return object.__new__(cls) +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C(1)) # revealed: C[int] # error: [invalid-assignment] "Object of type `C[int | str]` is not assignable to `C[int]`" @@ -279,11 +286,18 @@ wrong_innards: C[int] = C("five") ### `__init__` only ```py +from ty_extensions import generic_context, into_callable + class C[T]: x: T def __init__(self, x: T) -> None: ... +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C(1)) # revealed: C[int] # error: [invalid-assignment] "Object of type `C[int | str]` is not assignable to `C[int]`" @@ -293,6 +307,8 @@ wrong_innards: C[int] = C("five") ### Identical `__new__` and `__init__` signatures ```py +from ty_extensions import generic_context, into_callable + class C[T]: x: T @@ -301,6 +317,11 @@ class C[T]: def __init__(self, x: T) -> None: ... +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C(1)) # revealed: C[int] # error: [invalid-assignment] "Object of type `C[int | str]` is not assignable to `C[int]`" @@ -310,6 +331,8 @@ wrong_innards: C[int] = C("five") ### Compatible `__new__` and `__init__` signatures ```py +from ty_extensions import generic_context, into_callable + class C[T]: x: T @@ -318,6 +341,11 @@ class C[T]: def __init__(self, x: T) -> None: ... +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C(1)) # revealed: C[int] # error: [invalid-assignment] "Object of type `C[int | str]` is not assignable to `C[int]`" @@ -331,6 +359,11 @@ class D[T]: def __init__(self, *args, **kwargs) -> None: ... +# revealed: ty_extensions.GenericContext[T@D] +reveal_type(generic_context(D)) +# revealed: ty_extensions.GenericContext[T@D] +reveal_type(generic_context(into_callable(D))) + reveal_type(D(1)) # revealed: D[int] # error: [invalid-assignment] "Object of type `D[int | str]` is not assignable to `D[int]`" @@ -343,6 +376,8 @@ If either method comes from a generic base class, we don't currently use its inf to specialize the class. ```py +from ty_extensions import generic_context, into_callable + class C[T, U]: def __new__(cls, *args, **kwargs) -> "C[T, U]": return object.__new__(cls) @@ -350,18 +385,30 @@ class C[T, U]: class D[V](C[V, int]): def __init__(self, x: V) -> None: ... +# revealed: ty_extensions.GenericContext[V@D] +reveal_type(generic_context(D)) +# revealed: ty_extensions.GenericContext[V@D] +reveal_type(generic_context(into_callable(D))) + reveal_type(D(1)) # revealed: D[Literal[1]] ``` ### Generic class inherits `__init__` from generic base class ```py +from ty_extensions import generic_context, into_callable + class C[T, U]: def __init__(self, t: T, u: U) -> None: ... class D[T, U](C[T, U]): pass +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(D)) +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(into_callable(D))) + reveal_type(C(1, "str")) # revealed: C[Literal[1], Literal["str"]] reveal_type(D(1, "str")) # revealed: D[Literal[1], Literal["str"]] ``` @@ -371,9 +418,16 @@ reveal_type(D(1, "str")) # revealed: D[Literal[1], Literal["str"]] This is a specific example of the above, since it was reported specifically by a user. ```py +from ty_extensions import generic_context, into_callable + class D[T, U](dict[T, U]): pass +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(D)) +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(into_callable(D))) + reveal_type(D(key=1)) # revealed: D[str, int] ``` @@ -384,8 +438,15 @@ for `tuple`, so we use a different mechanism to make sure it has the right inher context. But from the user's point of view, this is another example of the above.) ```py +from ty_extensions import generic_context, into_callable + class C[T, U](tuple[T, U]): ... +# revealed: ty_extensions.GenericContext[T@C, U@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C, U@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C((1, 2))) # revealed: C[Literal[1], Literal[2]] ``` @@ -409,11 +470,18 @@ def func8(t1: tuple[complex, list[int]], t2: tuple[int, *tuple[str, ...]], t3: t ### `__init__` is itself generic ```py +from ty_extensions import generic_context, into_callable + class C[T]: x: T def __init__[S](self, x: T, y: S) -> None: ... +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C, S@__init__] +reveal_type(generic_context(into_callable(C))) + reveal_type(C(1, 1)) # revealed: C[int] reveal_type(C(1, "string")) # revealed: C[int] reveal_type(C(1, True)) # revealed: C[int] @@ -427,6 +495,7 @@ wrong_innards: C[int] = C("five", 1) ```py from __future__ import annotations from typing import overload +from ty_extensions import generic_context, into_callable class C[T]: # we need to use the type variable or else the class is bivariant in T, and @@ -443,6 +512,11 @@ class C[T]: def __init__(self, x: int) -> None: ... def __init__(self, x: str | bytes | int) -> None: ... +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C("string")) # revealed: C[str] reveal_type(C(b"bytes")) # revealed: C[bytes] reveal_type(C(12)) # revealed: C[Unknown] @@ -470,6 +544,11 @@ class D[T, U]: def __init__(self, t: T, u: U) -> None: ... def __init__(self, *args) -> None: ... +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(D)) +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(into_callable(D))) + reveal_type(D("string")) # revealed: D[str, Literal["string"]] reveal_type(D(1)) # revealed: D[str, Literal[1]] reveal_type(D(1, "string")) # revealed: D[Literal[1], Literal["string"]] @@ -479,24 +558,42 @@ reveal_type(D(1, "string")) # revealed: D[Literal[1], Literal["string"]] ```py from dataclasses import dataclass +from ty_extensions import generic_context, into_callable @dataclass class A[T]: x: T +# revealed: ty_extensions.GenericContext[T@A] +reveal_type(generic_context(A)) +# revealed: ty_extensions.GenericContext[T@A] +reveal_type(generic_context(into_callable(A))) + reveal_type(A(x=1)) # revealed: A[int] ``` ### Class typevar has another typevar as a default ```py +from ty_extensions import generic_context, into_callable + class C[T, U = T]: ... +# revealed: ty_extensions.GenericContext[T@C, U@C] +reveal_type(generic_context(C)) +# revealed: ty_extensions.GenericContext[T@C, U@C] +reveal_type(generic_context(into_callable(C))) + reveal_type(C()) # revealed: C[Unknown, Unknown] class D[T, U = T]: def __init__(self) -> None: ... +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(D)) +# revealed: ty_extensions.GenericContext[T@D, U@D] +reveal_type(generic_context(into_callable(D))) + reveal_type(D()) # revealed: D[Unknown, Unknown] ``` diff --git a/crates/ty_python_semantic/resources/mdtest/liskov.md b/crates/ty_python_semantic/resources/mdtest/liskov.md index ad16b99f08..2dca2dd558 100644 --- a/crates/ty_python_semantic/resources/mdtest/liskov.md +++ b/crates/ty_python_semantic/resources/mdtest/liskov.md @@ -218,8 +218,8 @@ class E(A[int]): def method(self, x: object) -> None: ... # fine class F[T](A[T]): - # TODO: we should emit `invalid-method-override` on this: # `str` is not necessarily a supertype of `T`! + # error: [invalid-method-override] def method(self, x: str) -> None: ... class G(A[int]): diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 61cc160769..6022aad53e 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -8535,12 +8535,9 @@ impl<'db> TypeMapping<'_, 'db> { | TypeMapping::Materialize(_) | TypeMapping::ReplaceParameterDefaults | TypeMapping::EagerExpansion => context, - TypeMapping::BindSelf { .. } => GenericContext::from_typevar_instances( - db, - context - .variables(db) - .filter(|var| !var.typevar(db).is_self(db)), - ), + TypeMapping::BindSelf { + binding_context, .. + } => context.remove_self(db, *binding_context), TypeMapping::ReplaceSelf { new_upper_bound } => GenericContext::from_typevar_instances( db, context.variables(db).map(|typevar| { diff --git a/crates/ty_python_semantic/src/types/call/bind.rs b/crates/ty_python_semantic/src/types/call/bind.rs index 2093f2f377..da63e45208 100644 --- a/crates/ty_python_semantic/src/types/call/bind.rs +++ b/crates/ty_python_semantic/src/types/call/bind.rs @@ -32,7 +32,9 @@ use crate::types::function::{ use crate::types::generics::{ InferableTypeVars, Specialization, SpecializationBuilder, SpecializationError, }; -use crate::types::signatures::{Parameter, ParameterForm, ParameterKind, Parameters}; +use crate::types::signatures::{ + CallableSignature, Parameter, ParameterForm, ParameterKind, Parameters, +}; use crate::types::tuple::{TupleLength, TupleType}; use crate::types::{ BoundMethodType, BoundTypeVarIdentity, ClassLiteral, DATACLASS_FLAGS, DataclassFlags, @@ -788,51 +790,67 @@ impl<'db> Bindings<'db> { )) }; - let function_generic_context = |function: FunctionType<'db>| { - let union = UnionType::from_elements( - db, - function - .signature(db) - .overloads - .iter() - .filter_map(|signature| signature.generic_context) - .map(wrap_generic_context), - ); - if union.is_never() { - Type::none(db) - } else { - union - } - }; + let signature_generic_context = + |signature: &CallableSignature<'db>| { + UnionType::try_from_elements( + db, + signature.overloads.iter().map(|signature| { + signature.generic_context.map(wrap_generic_context) + }), + ) + }; - // TODO: Handle generic functions, and unions/intersections of - // generic types - overload.set_return_type(match ty { - Type::ClassLiteral(class) => class - .generic_context(db) - .map(wrap_generic_context) - .unwrap_or_else(|| Type::none(db)), + let generic_context_for_simple_type = |ty: Type<'db>| match ty { + Type::ClassLiteral(class) => { + class.generic_context(db).map(wrap_generic_context) + } Type::FunctionLiteral(function) => { - function_generic_context(*function) + signature_generic_context(function.signature(db)) } - Type::BoundMethod(bound_method) => { - function_generic_context(bound_method.function(db)) + Type::BoundMethod(bound_method) => signature_generic_context( + bound_method.function(db).signature(db), + ), + + Type::Callable(callable) => { + signature_generic_context(callable.signatures(db)) } Type::KnownInstance(KnownInstanceType::TypeAliasType( TypeAliasType::PEP695(alias), - )) => alias - .generic_context(db) - .map(wrap_generic_context) - .unwrap_or_else(|| Type::none(db)), + )) => alias.generic_context(db).map(wrap_generic_context), - _ => Type::none(db), - }); + _ => None, + }; + + let generic_context = match ty { + Type::Union(union_type) => UnionType::try_from_elements( + db, + union_type + .elements(db) + .iter() + .map(|ty| generic_context_for_simple_type(*ty)), + ), + _ => generic_context_for_simple_type(*ty), + }; + + overload.set_return_type( + generic_context.unwrap_or_else(|| Type::none(db)), + ); } } + Some(KnownFunction::IntoCallable) => { + let [Some(ty)] = overload.parameter_types() else { + continue; + }; + let Some(callables) = ty.try_upcast_to_callable(db) else { + continue; + }; + overload.set_return_type(callables.into_type(db)); + } + Some(KnownFunction::DunderAllNames) => { if let [Some(ty)] = overload.parameter_types() { overload.set_return_type(match ty { diff --git a/crates/ty_python_semantic/src/types/class.rs b/crates/ty_python_semantic/src/types/class.rs index 06f91502fc..00b5bed3ec 100644 --- a/crates/ty_python_semantic/src/types/class.rs +++ b/crates/ty_python_semantic/src/types/class.rs @@ -1133,6 +1133,13 @@ impl<'db> ClassType<'db> { /// constructor signature of this class. #[salsa::tracked(cycle_initial=into_callable_cycle_initial, heap_size=ruff_memory_usage::heap_size)] pub(super) fn into_callable(self, db: &'db dyn Db) -> CallableTypes<'db> { + // TODO: This mimics a lot of the logic in Type::try_call_from_constructor. Can we + // consolidate the two? Can we invoke a class by upcasting the class into a Callable, and + // then relying on the call binding machinery to Just Work™? + + let (class_literal, _) = self.class_literal(db); + let class_generic_context = class_literal.generic_context(db); + let self_ty = Type::from(self); let metaclass_dunder_call_function_symbol = self_ty .member_lookup_with_policy( @@ -1206,39 +1213,58 @@ impl<'db> ClassType<'db> { // If the class defines an `__init__` method, then we synthesize a callable type with the // same parameters as the `__init__` method after it is bound, and with the return type of // the concrete type of `Self`. - let synthesized_dunder_init_callable = - if let Place::Defined(ty, _, _) = dunder_init_function_symbol { - let signature = match ty { - Type::FunctionLiteral(dunder_init_function) => { - Some(dunder_init_function.signature(db)) - } - Type::Callable(callable) => Some(callable.signatures(db)), - _ => None, + let synthesized_dunder_init_callable = if let Place::Defined(ty, _, _) = + dunder_init_function_symbol + { + let signature = match ty { + Type::FunctionLiteral(dunder_init_function) => { + Some(dunder_init_function.signature(db)) + } + Type::Callable(callable) => Some(callable.signatures(db)), + _ => None, + }; + + if let Some(signature) = signature { + let synthesized_signature = |signature: &Signature<'db>| { + let self_annotation = signature + .parameters() + .get_positional(0) + .and_then(Parameter::annotated_type) + .filter(|ty| { + ty.as_typevar() + .is_none_or(|bound_typevar| !bound_typevar.typevar(db).is_self(db)) + }); + let return_type = self_annotation.unwrap_or(correct_return_type); + let instance_ty = self_annotation.unwrap_or_else(|| Type::instance(db, self)); + let generic_context = GenericContext::merge_optional( + db, + class_generic_context, + signature.generic_context, + ); + Signature::new_generic( + generic_context, + signature.parameters().clone(), + Some(return_type), + ) + .with_definition(signature.definition()) + .bind_self(db, Some(instance_ty)) }; - if let Some(signature) = signature { - let synthesized_signature = |signature: &Signature<'db>| { - let instance_ty = Type::instance(db, self); - Signature::new(signature.parameters().clone(), Some(correct_return_type)) - .with_definition(signature.definition()) - .bind_self(db, Some(instance_ty)) - }; + let synthesized_dunder_init_signature = CallableSignature::from_overloads( + signature.overloads.iter().map(synthesized_signature), + ); - let synthesized_dunder_init_signature = CallableSignature::from_overloads( - signature.overloads.iter().map(synthesized_signature), - ); - - Some(CallableType::new( - db, - synthesized_dunder_init_signature, - true, - )) - } else { - None - } + Some(CallableType::new( + db, + synthesized_dunder_init_signature, + true, + )) } else { None - }; + } + } else { + None + }; match (dunder_new_function, synthesized_dunder_init_callable) { (Some(dunder_new_function), Some(synthesized_dunder_init_callable)) => { @@ -1261,9 +1287,13 @@ impl<'db> ClassType<'db> { ) .place; - if let Place::Defined(Type::FunctionLiteral(new_function), _, _) = + if let Place::Defined(Type::FunctionLiteral(mut new_function), _, _) = new_function_symbol { + if let Some(class_generic_context) = class_generic_context { + new_function = + new_function.with_inherited_generic_context(db, class_generic_context); + } CallableTypes::one( new_function .into_bound_method_type(db, correct_return_type) @@ -1273,7 +1303,11 @@ impl<'db> ClassType<'db> { // Fallback if no `object.__new__` is found. CallableTypes::one(CallableType::single( db, - Signature::new(Parameters::empty(), Some(correct_return_type)), + Signature::new_generic( + class_generic_context, + Parameters::empty(), + Some(correct_return_type), + ), )) } } diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index 0fba3aecd8..7380ec86f3 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -1339,6 +1339,8 @@ pub enum KnownFunction { IsSingleValued, /// `ty_extensions.generic_context` GenericContext, + /// `ty_extensions.into_callable` + IntoCallable, /// `ty_extensions.dunder_all_names` DunderAllNames, /// `ty_extensions.enum_members` @@ -1411,6 +1413,7 @@ impl KnownFunction { | Self::IsSingleton | Self::IsSubtypeOf | Self::GenericContext + | Self::IntoCallable | Self::DunderAllNames | Self::EnumMembers | Self::StaticAssert @@ -1946,6 +1949,7 @@ pub(crate) mod tests { KnownFunction::IsSingleton | KnownFunction::IsSubtypeOf | KnownFunction::GenericContext + | KnownFunction::IntoCallable | KnownFunction::DunderAllNames | KnownFunction::EnumMembers | KnownFunction::StaticAssert diff --git a/crates/ty_python_semantic/src/types/generics.rs b/crates/ty_python_semantic/src/types/generics.rs index de357dfbce..432785b778 100644 --- a/crates/ty_python_semantic/src/types/generics.rs +++ b/crates/ty_python_semantic/src/types/generics.rs @@ -17,11 +17,12 @@ use crate::types::signatures::Parameters; use crate::types::tuple::{TupleSpec, TupleType, walk_tuple_type}; use crate::types::visitor::{TypeCollector, TypeVisitor, walk_type_with_recursion_guard}; use crate::types::{ - ApplyTypeMappingVisitor, BoundTypeVarIdentity, BoundTypeVarInstance, ClassLiteral, - FindLegacyTypeVarsVisitor, HasRelationToVisitor, IsDisjointVisitor, IsEquivalentVisitor, - KnownClass, KnownInstanceType, MaterializationKind, NormalizedVisitor, Type, TypeContext, - TypeMapping, TypeRelation, TypeVarBoundOrConstraints, TypeVarIdentity, TypeVarInstance, - TypeVarKind, TypeVarVariance, UnionType, declaration_type, walk_bound_type_var_type, + ApplyTypeMappingVisitor, BindingContext, BoundTypeVarIdentity, BoundTypeVarInstance, + ClassLiteral, FindLegacyTypeVarsVisitor, HasRelationToVisitor, IsDisjointVisitor, + IsEquivalentVisitor, KnownClass, KnownInstanceType, MaterializationKind, NormalizedVisitor, + Type, TypeContext, TypeMapping, TypeRelation, TypeVarBoundOrConstraints, TypeVarIdentity, + TypeVarInstance, TypeVarKind, TypeVarVariance, UnionType, declaration_type, + walk_bound_type_var_type, }; use crate::{Db, FxOrderMap, FxOrderSet}; @@ -261,6 +262,34 @@ impl<'db> GenericContext<'db> { ) } + pub(crate) fn merge_optional( + db: &'db dyn Db, + left: Option, + right: Option, + ) -> Option { + match (left, right) { + (None, None) => None, + (Some(one), None) | (None, Some(one)) => Some(one), + (Some(left), Some(right)) => Some(left.merge(db, right)), + } + } + + pub(crate) fn remove_self( + self, + db: &'db dyn Db, + binding_context: Option>, + ) -> Self { + Self::from_typevar_instances( + db, + self.variables(db).filter(|bound_typevar| { + !(bound_typevar.typevar(db).is_self(db) + && binding_context.is_none_or(|binding_context| { + bound_typevar.binding_context(db) == binding_context + })) + }), + ) + } + pub(crate) fn inferable_typevars(self, db: &'db dyn Db) -> InferableTypeVars<'db, 'db> { #[derive(Default)] struct CollectTypeVars<'db> { diff --git a/crates/ty_python_semantic/src/types/signatures.rs b/crates/ty_python_semantic/src/types/signatures.rs index d091b8a53f..c76a086cfa 100644 --- a/crates/ty_python_semantic/src/types/signatures.rs +++ b/crates/ty_python_semantic/src/types/signatures.rs @@ -667,10 +667,11 @@ impl<'db> Signature<'db> { let mut parameters = Parameters::new(db, parameters); let mut return_ty = self.return_ty; + let binding_context = self.definition.map(BindingContext::Definition); if let Some(self_type) = self_type { let self_mapping = TypeMapping::BindSelf { self_type, - binding_context: self.definition.map(BindingContext::Definition), + binding_context, }; parameters = parameters.apply_type_mapping_impl( db, @@ -682,7 +683,9 @@ impl<'db> Signature<'db> { .map(|ty| ty.apply_type_mapping(db, &self_mapping, TypeContext::default())); } Self { - generic_context: self.generic_context, + generic_context: self + .generic_context + .map(|generic_context| generic_context.remove_self(db, binding_context)), definition: self.definition, parameters, return_ty, diff --git a/crates/ty_vendored/ty_extensions/ty_extensions.pyi b/crates/ty_vendored/ty_extensions/ty_extensions.pyi index 8f45b29238..347b6b4b34 100644 --- a/crates/ty_vendored/ty_extensions/ty_extensions.pyi +++ b/crates/ty_vendored/ty_extensions/ty_extensions.pyi @@ -147,6 +147,10 @@ def is_single_valued(ty: Any) -> bool: # type is not generic. def generic_context(ty: Any) -> GenericContext | None: ... +# Converts a value into a `Callable`, if possible. This is the value equivalent +# of `CallableTypeOf`, which operates on types. +def into_callable(ty: Any) -> Any: ... + # Returns the `__all__` names of a module as a tuple of sorted strings, or `None` if # either the module does not have `__all__` or it has invalid elements. def dunder_all_names(module: Any) -> Any: ...