Python
/
ruff
mirror of https://github.com/astral-sh/ruff
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[red-knot] Statically known branches (#15019) 

## Summary

This changeset adds support for precise type-inference and
boundness-handling of definitions inside control-flow branches with
statically-known conditions, i.e. test-expressions whose truthiness we
can unambiguously infer as *always false* or *always true*.

This branch also includes:
- `sys.platform` support
- statically-known branches handling for Boolean expressions and while
  loops
- new `target-version` requirements in some Markdown tests which were
  now required due to the understanding of `sys.version_info` branches.

closes #12700 
closes #15034 

## Performance

### `tomllib`, -7%, needs to resolve one additional module (sys)

| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
|:---|---:|---:|---:|---:|
| `./red_knot_main --project /home/shark/tomllib` | 22.2 ± 1.3 | 19.1 |
25.6 | 1.00 |
| `./red_knot_feature --project /home/shark/tomllib` | 23.8 ± 1.6 | 20.8
| 28.6 | 1.07 ± 0.09 |

### `black`, -6%

| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
|:---|---:|---:|---:|---:|
| `./red_knot_main --project /home/shark/black` | 129.3 ± 5.1 | 119.0 |
137.8 | 1.00 |
| `./red_knot_feature --project /home/shark/black` | 136.5 ± 6.8 | 123.8
| 147.5 | 1.06 ± 0.07 |

## Test Plan

- New Markdown tests for the main feature in
  `statically-known-branches.md`
- New Markdown tests for `sys.platform`
- Adapted tests for `EllipsisType`, `Never`, etc
This commit is contained in:
David Peter 2024-12-21 11:33:10 +01:00 committed by GitHub
parent d3f51cf3a6
commit 000948ad3b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
39 changed files with 3187 additions and 632 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
crates/red_knot_python_semantic/resources/mdtest/annotations/never.md
View File

@ -47,7 +47,9 @@ def f():
## `typing.Never` ## `typing.Never`
`typing.Never` is only available in Python 3.11 and later: `typing.Never` is only available in Python 3.11 and later.
### Python 3.11
```toml ```toml
[environment] [environment]
@ -57,8 +59,17 @@ python-version = "3.11"
```py ```py
from typing import Never from typing import Never
x: Never reveal_type(Never) # revealed: typing.Never
```
def f():
reveal_type(x) # revealed: Never ### Python 3.10
```toml
[environment]
python-version = "3.10"
```
```py
# error: [unresolved-import]
from typing import Never
``` ```

2
crates/red_knot_python_semantic/resources/mdtest/assignment/annotations.md
View File

@ -33,8 +33,6 @@ b: tuple[int] = (42,)
c: tuple[str, int] = ("42", 42) c: tuple[str, int] = ("42", 42)
d: tuple[tuple[str, str], tuple[int, int]] = (("foo", "foo"), (42, 42)) d: tuple[tuple[str, str], tuple[int, int]] = (("foo", "foo"), (42, 42))
e: tuple[str, ...] = () e: tuple[str, ...] = ()
# TODO: we should not emit this error
# error: [call-possibly-unbound-method] "Method `__class_getitem__` of type `Literal[tuple]` is possibly unbound"
f: tuple[str, *tuple[int, ...], bytes] = ("42", b"42") f: tuple[str, *tuple[int, ...], bytes] = ("42", b"42")
g: tuple[str, Unpack[tuple[int, ...]], bytes] = ("42", b"42") g: tuple[str, Unpack[tuple[int, ...]], bytes] = ("42", b"42")
h: tuple[list[int], list[int]] = ([], []) h: tuple[list[int], list[int]] = ([], [])

7
crates/red_knot_python_semantic/resources/mdtest/boolean/short_circuit.md
View File

@ -32,13 +32,10 @@ def _(flag: bool):
```py ```py
if True or (x := 1): if True or (x := 1):
# TODO: infer that the second arm is never executed, and raise `unresolved-reference`. # error: [unresolved-reference]
# error: [possibly-unresolved-reference] reveal_type(x) # revealed: Unknown
reveal_type(x) # revealed: Literal[1]
if True and (x := 1): if True and (x := 1):
# TODO: infer that the second arm is always executed, do not raise a diagnostic
# error: [possibly-unresolved-reference]
reveal_type(x) # revealed: Literal[1] reveal_type(x) # revealed: Literal[1]
``` ```

4
crates/red_knot_python_semantic/resources/mdtest/call/callable_instance.md
View File

@ -67,6 +67,6 @@ def _(flag: bool):
def __call__(self) -> int: ... def __call__(self) -> int: ...
a = NonCallable() a = NonCallable()
# error: "Object of type `Literal[__call__] | Literal[1]` is not callable (due to union element `Literal[1]`)" # error: "Object of type `Literal[1] | Literal[__call__]` is not callable (due to union element `Literal[1]`)"
reveal_type(a()) # revealed: int | Unknown reveal_type(a()) # revealed: Unknown | int
``` ```

2
crates/red_knot_python_semantic/resources/mdtest/expression/if.md
View File

@ -7,7 +7,7 @@ def _(flag: bool):
reveal_type(1 if flag else 2) # revealed: Literal[1, 2] reveal_type(1 if flag else 2) # revealed: Literal[1, 2]
``` ```
## Statically known branches ## Statically known conditions in if-expressions
```py ```py
reveal_type(1 if True else 2) # revealed: Literal[1] reveal_type(1 if True else 2) # revealed: Literal[1]

20
crates/red_knot_python_semantic/resources/mdtest/literal/ellipsis.md
View File

@ -1,7 +1,23 @@
# Ellipsis literals # Ellipsis literals
## Simple ## Python 3.9
```toml
[environment]
python-version = "3.9"
```
```py ```py
reveal_type(...) # revealed: EllipsisType | ellipsis reveal_type(...) # revealed: ellipsis
```
## Python 3.10
```toml
[environment]
python-version = "3.10"
```
```py
reveal_type(...) # revealed: EllipsisType
``` ```

10
crates/red_knot_python_semantic/resources/mdtest/narrow/issubclass.md
View File

@ -95,10 +95,14 @@ def _(t: type[object]):
### Handling of `None` ### Handling of `None`
`types.NoneType` is only available in Python 3.10 and later:
```toml
[environment]
python-version = "3.10"
```
```py ```py
# TODO: this error should ideally go away once we (1) understand `sys.version_info` branches,
# and (2) set the target Python version for this test to 3.10.
# error: [possibly-unbound-import] "Member `NoneType` of module `types` is possibly unbound"
from types import NoneType from types import NoneType
def _(flag: bool): def _(flag: bool):

1530
crates/red_knot_python_semantic/resources/mdtest/statically_known_branches.md Normal file
View File

File diff suppressed because it is too large Load Diff

5
crates/red_knot_python_semantic/resources/mdtest/subscript/tuple.md
View File

@ -81,10 +81,7 @@ python-version = "3.9"
``` ```
```py ```py
# TODO: # TODO: `tuple[int, str]` is a valid base (generics)
# * `tuple.__class_getitem__` is always bound on 3.9 (`sys.version_info`)
# * `tuple[int, str]` is a valid base (generics)
# error: [call-possibly-unbound-method] "Method `__class_getitem__` of type `Literal[tuple]` is possibly unbound"
# error: [invalid-base] "Invalid class base with type `GenericAlias` (all bases must be a class, `Any`, `Unknown` or `Todo`)" # error: [invalid-base] "Invalid class base with type `GenericAlias` (all bases must be a class, `Any`, `Unknown` or `Todo`)"
class A(tuple[int, str]): ... class A(tuple[int, str]): ...

71
crates/red_knot_python_semantic/resources/mdtest/sys_platform.md Normal file
View File

@ -0,0 +1,71 @@
# `sys.platform`
## Default value
When no target platform is specified, we fall back to the type of `sys.platform` declared in
typeshed:
```toml
[environment]
# No python-platform entry
```
```py
import sys
reveal_type(sys.platform) # revealed: str
```
## Explicit selection of `all` platforms
```toml
[environment]
python-platform = "all"
```
```py
import sys
reveal_type(sys.platform) # revealed: str
```
## Explicit selection of a specific platform
```toml
[environment]
python-platform = "linux"
```
```py
import sys
reveal_type(sys.platform) # revealed: Literal["linux"]
```
## Testing for a specific platform
### Exact comparison
```toml
[environment]
python-platform = "freebsd8"
```
```py
import sys
reveal_type(sys.platform == "freebsd8") # revealed: Literal[True]
reveal_type(sys.platform == "linux") # revealed: Literal[False]
```
### Substring comparison
It is [recommended](https://docs.python.org/3/library/sys.html#sys.platform) to use
`sys.platform.startswith(...)` for platform checks. This is not yet supported in type inference:
```py
import sys
reveal_type(sys.platform.startswith("freebsd")) # revealed: @Todo(instance attributes)
reveal_type(sys.platform.startswith("linux")) # revealed: @Todo(instance attributes)
```

6
crates/red_knot_python_semantic/src/db.rs
View File

@ -16,7 +16,7 @@ pub(crate) mod tests {
use crate::program::{Program, SearchPathSettings}; use crate::program::{Program, SearchPathSettings};
use crate::python_version::PythonVersion; use crate::python_version::PythonVersion;
use crate::{default_lint_registry, ProgramSettings}; use crate::{default_lint_registry, ProgramSettings, PythonPlatform};
use super::Db; use super::Db;
use crate::lint::RuleSelection; use crate::lint::RuleSelection;
@ -127,6 +127,8 @@ pub(crate) mod tests {
pub(crate) struct TestDbBuilder<'a> { pub(crate) struct TestDbBuilder<'a> {
/// Target Python version /// Target Python version
python_version: PythonVersion, python_version: PythonVersion,
/// Target Python platform
python_platform: PythonPlatform,
/// Path to a custom typeshed directory /// Path to a custom typeshed directory
custom_typeshed: Option<SystemPathBuf>, custom_typeshed: Option<SystemPathBuf>,
/// Path and content pairs for files that should be present /// Path and content pairs for files that should be present
@ -137,6 +139,7 @@ pub(crate) mod tests {
pub(crate) fn new() -> Self { pub(crate) fn new() -> Self {
Self { Self {
python_version: PythonVersion::default(), python_version: PythonVersion::default(),
python_platform: PythonPlatform::default(),
custom_typeshed: None, custom_typeshed: None,
files: vec![], files: vec![],
} }
@ -173,6 +176,7 @@ pub(crate) mod tests {
&db, &db,
&ProgramSettings { &ProgramSettings {
python_version: self.python_version, python_version: self.python_version,
python_platform: self.python_platform,
search_paths, search_paths,
}, },
) )

3
crates/red_knot_python_semantic/src/lib.rs
View File

@ -7,6 +7,7 @@ pub use db::Db;
pub use module_name::ModuleName; pub use module_name::ModuleName;
pub use module_resolver::{resolve_module, system_module_search_paths, KnownModule, Module}; pub use module_resolver::{resolve_module, system_module_search_paths, KnownModule, Module};
pub use program::{Program, ProgramSettings, SearchPathSettings, SitePackages}; pub use program::{Program, ProgramSettings, SearchPathSettings, SitePackages};
pub use python_platform::PythonPlatform;
pub use python_version::PythonVersion; pub use python_version::PythonVersion;
pub use semantic_model::{HasTy, SemanticModel}; pub use semantic_model::{HasTy, SemanticModel};
@ -17,6 +18,7 @@ mod module_name;
mod module_resolver; mod module_resolver;
mod node_key; mod node_key;
mod program; mod program;
mod python_platform;
mod python_version; mod python_version;
pub mod semantic_index; pub mod semantic_index;
mod semantic_model; mod semantic_model;
@ -27,6 +29,7 @@ pub(crate) mod symbol;
pub mod types; pub mod types;
mod unpack; mod unpack;
mod util; mod util;
mod visibility_constraints;
type FxOrderSet<V> = ordermap::set::OrderSet<V, BuildHasherDefault<FxHasher>>; type FxOrderSet<V> = ordermap::set::OrderSet<V, BuildHasherDefault<FxHasher>>;

6
crates/red_knot_python_semantic/src/module_resolver/resolver.rs
View File

@ -721,8 +721,8 @@ mod tests {
use crate::module_name::ModuleName; use crate::module_name::ModuleName;
use crate::module_resolver::module::ModuleKind; use crate::module_resolver::module::ModuleKind;
use crate::module_resolver::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder}; use crate::module_resolver::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
use crate::ProgramSettings;
use crate::PythonVersion; use crate::PythonVersion;
use crate::{ProgramSettings, PythonPlatform};
use super::*; use super::*;
@ -1262,7 +1262,7 @@ mod tests {
fn symlink() -> anyhow::Result<()> { fn symlink() -> anyhow::Result<()> {
use anyhow::Context; use anyhow::Context;
use crate::program::Program; use crate::{program::Program, PythonPlatform};
use ruff_db::system::{OsSystem, SystemPath}; use ruff_db::system::{OsSystem, SystemPath};
use crate::db::tests::TestDb; use crate::db::tests::TestDb;
@ -1296,6 +1296,7 @@ mod tests {
&db, &db,
&ProgramSettings { &ProgramSettings {
python_version: PythonVersion::PY38, python_version: PythonVersion::PY38,
python_platform: PythonPlatform::default(),
search_paths: SearchPathSettings { search_paths: SearchPathSettings {
extra_paths: vec![], extra_paths: vec![],
src_root: src.clone(), src_root: src.clone(),
@ -1801,6 +1802,7 @@ not_a_directory
&db, &db,
&ProgramSettings { &ProgramSettings {
python_version: PythonVersion::default(), python_version: PythonVersion::default(),
python_platform: PythonPlatform::default(),
search_paths: SearchPathSettings { search_paths: SearchPathSettings {
extra_paths: vec![], extra_paths: vec![],
src_root: SystemPathBuf::from("/src"), src_root: SystemPathBuf::from("/src"),

12
crates/red_knot_python_semantic/src/module_resolver/testing.rs
View File

@ -4,7 +4,7 @@ use ruff_db::vendored::VendoredPathBuf;
use crate::db::tests::TestDb; use crate::db::tests::TestDb;
use crate::program::{Program, SearchPathSettings}; use crate::program::{Program, SearchPathSettings};
use crate::python_version::PythonVersion; use crate::python_version::PythonVersion;
use crate::{ProgramSettings, SitePackages}; use crate::{ProgramSettings, PythonPlatform, SitePackages};
/// A test case for the module resolver. /// A test case for the module resolver.
/// ///
@ -101,6 +101,7 @@ pub(crate) struct UnspecifiedTypeshed;
pub(crate) struct TestCaseBuilder<T> { pub(crate) struct TestCaseBuilder<T> {
typeshed_option: T, typeshed_option: T,
python_version: PythonVersion, python_version: PythonVersion,
python_platform: PythonPlatform,
first_party_files: Vec<FileSpec>, first_party_files: Vec<FileSpec>,
site_packages_files: Vec<FileSpec>, site_packages_files: Vec<FileSpec>,
} }
@ -147,6 +148,7 @@ impl TestCaseBuilder<UnspecifiedTypeshed> {
Self { Self {
typeshed_option: UnspecifiedTypeshed, typeshed_option: UnspecifiedTypeshed,
python_version: PythonVersion::default(), python_version: PythonVersion::default(),
python_platform: PythonPlatform::default(),
first_party_files: vec![], first_party_files: vec![],
site_packages_files: vec![], site_packages_files: vec![],
} }
@ -157,12 +159,14 @@ impl TestCaseBuilder<UnspecifiedTypeshed> {
let TestCaseBuilder { let TestCaseBuilder {
typeshed_option: _, typeshed_option: _,
python_version, python_version,
python_platform,
first_party_files, first_party_files,
site_packages_files, site_packages_files,
} = self; } = self;
TestCaseBuilder { TestCaseBuilder {
typeshed_option: VendoredTypeshed, typeshed_option: VendoredTypeshed,
python_version, python_version,
python_platform,
first_party_files, first_party_files,
site_packages_files, site_packages_files,
} }
@ -176,6 +180,7 @@ impl TestCaseBuilder<UnspecifiedTypeshed> {
let TestCaseBuilder { let TestCaseBuilder {
typeshed_option: _, typeshed_option: _,
python_version, python_version,
python_platform,
first_party_files, first_party_files,
site_packages_files, site_packages_files,
} = self; } = self;
@ -183,6 +188,7 @@ impl TestCaseBuilder<UnspecifiedTypeshed> {
TestCaseBuilder { TestCaseBuilder {
typeshed_option: typeshed, typeshed_option: typeshed,
python_version, python_version,
python_platform,
first_party_files, first_party_files,
site_packages_files, site_packages_files,
} }
@ -212,6 +218,7 @@ impl TestCaseBuilder<MockedTypeshed> {
let TestCaseBuilder { let TestCaseBuilder {
typeshed_option, typeshed_option,
python_version, python_version,
python_platform,
first_party_files, first_party_files,
site_packages_files, site_packages_files,
} = self; } = self;
@ -227,6 +234,7 @@ impl TestCaseBuilder<MockedTypeshed> {
&db, &db,
&ProgramSettings { &ProgramSettings {
python_version, python_version,
python_platform,
search_paths: SearchPathSettings { search_paths: SearchPathSettings {
extra_paths: vec![], extra_paths: vec![],
src_root: src.clone(), src_root: src.clone(),
@ -269,6 +277,7 @@ impl TestCaseBuilder<VendoredTypeshed> {
let TestCaseBuilder { let TestCaseBuilder {
typeshed_option: VendoredTypeshed, typeshed_option: VendoredTypeshed,
python_version, python_version,
python_platform,
first_party_files, first_party_files,
site_packages_files, site_packages_files,
} = self; } = self;
@ -283,6 +292,7 @@ impl TestCaseBuilder<VendoredTypeshed> {
&db, &db,
&ProgramSettings { &ProgramSettings {
python_version, python_version,
python_platform,
search_paths: SearchPathSettings { search_paths: SearchPathSettings {
site_packages: SitePackages::Known(vec![site_packages.clone()]), site_packages: SitePackages::Known(vec![site_packages.clone()]),
..SearchPathSettings::new(src.clone()) ..SearchPathSettings::new(src.clone())

13
crates/red_knot_python_semantic/src/program.rs
View File

@ -1,3 +1,4 @@
use crate::python_platform::PythonPlatform;
use crate::python_version::PythonVersion; use crate::python_version::PythonVersion;
use anyhow::Context; use anyhow::Context;
use salsa::Durability; use salsa::Durability;
@ -12,6 +13,8 @@ use crate::Db;
pub struct Program { pub struct Program {
pub python_version: PythonVersion, pub python_version: PythonVersion,
pub python_platform: PythonPlatform,
#[return_ref] #[return_ref]
pub(crate) search_paths: SearchPaths, pub(crate) search_paths: SearchPaths,
} }
@ -20,6 +23,7 @@ impl Program {
pub fn from_settings(db: &dyn Db, settings: &ProgramSettings) -> anyhow::Result<Self> { pub fn from_settings(db: &dyn Db, settings: &ProgramSettings) -> anyhow::Result<Self> {
let ProgramSettings { let ProgramSettings {
python_version, python_version,
python_platform,
search_paths, search_paths,
} = settings; } = settings;
@ -28,9 +32,11 @@ impl Program {
let search_paths = SearchPaths::from_settings(db, search_paths) let search_paths = SearchPaths::from_settings(db, search_paths)
.with_context(|| "Invalid search path settings")?; .with_context(|| "Invalid search path settings")?;
Ok(Program::builder(settings.python_version, search_paths) Ok(
.durability(Durability::HIGH) Program::builder(*python_version, python_platform.clone(), search_paths)
.new(db)) .durability(Durability::HIGH)
.new(db),
)
} }
pub fn update_search_paths( pub fn update_search_paths(
@ -57,6 +63,7 @@ impl Program {
#[cfg_attr(feature = "serde", derive(serde::Serialize))] #[cfg_attr(feature = "serde", derive(serde::Serialize))]
pub struct ProgramSettings { pub struct ProgramSettings {
pub python_version: PythonVersion, pub python_version: PythonVersion,
pub python_platform: PythonPlatform,
pub search_paths: SearchPathSettings, pub search_paths: SearchPathSettings,
} }

19
crates/red_knot_python_semantic/src/python_platform.rs Normal file
View File

@ -0,0 +1,19 @@
/// The target platform to assume when resolving types.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[cfg_attr(
feature = "serde",
derive(serde::Serialize, serde::Deserialize),
serde(rename_all = "kebab-case")
)]
pub enum PythonPlatform {
/// Do not make any assumptions about the target platform.
#[default]
All,
/// Assume a specific target platform like `linux`, `darwin` or `win32`.
///
/// We use a string (instead of individual enum variants), as the set of possible platforms
/// may change over time. See <https://docs.python.org/3/library/sys.html#sys.platform> for
/// some known platform identifiers.
#[cfg_attr(feature = "serde", serde(untagged))]
Identifier(String),
}

9
crates/red_knot_python_semantic/src/semantic_index.rs
View File

@ -29,7 +29,8 @@ pub mod symbol;
mod use_def; mod use_def;
pub(crate) use self::use_def::{ pub(crate) use self::use_def::{
BindingWithConstraints, BindingWithConstraintsIterator, DeclarationsIterator, BindingWithConstraints, BindingWithConstraintsIterator, DeclarationWithConstraint,
DeclarationsIterator, ScopedVisibilityConstraintId,
}; };
type SymbolMap = hashbrown::HashMap<ScopedSymbolId, (), FxBuildHasher>; type SymbolMap = hashbrown::HashMap<ScopedSymbolId, (), FxBuildHasher>;
@ -378,14 +379,12 @@ mod tests {
impl UseDefMap<'_> { impl UseDefMap<'_> {
fn first_public_binding(&self, symbol: ScopedSymbolId) -> Option<Definition<'_>> { fn first_public_binding(&self, symbol: ScopedSymbolId) -> Option<Definition<'_>> {
self.public_bindings(symbol) self.public_bindings(symbol)
.next() .find_map(|constrained_binding| constrained_binding.binding)
.map(|constrained_binding| constrained_binding.binding)
} }
fn first_binding_at_use(&self, use_id: ScopedUseId) -> Option<Definition<'_>> { fn first_binding_at_use(&self, use_id: ScopedUseId) -> Option<Definition<'_>> {
self.bindings_at_use(use_id) self.bindings_at_use(use_id)
.next() .find_map(|constrained_binding| constrained_binding.binding)
.map(|constrained_binding| constrained_binding.binding)
} }
} }

277
crates/red_knot_python_semantic/src/semantic_index/builder.rs
View File

@ -6,15 +6,16 @@ use rustc_hash::{FxHashMap, FxHashSet};
use ruff_db::files::File; use ruff_db::files::File;
use ruff_db::parsed::ParsedModule; use ruff_db::parsed::ParsedModule;
use ruff_index::IndexVec; use ruff_index::IndexVec;
use ruff_python_ast as ast;
use ruff_python_ast::name::Name; use ruff_python_ast::name::Name;
use ruff_python_ast::visitor::{walk_expr, walk_pattern, walk_stmt, Visitor}; use ruff_python_ast::visitor::{walk_expr, walk_pattern, walk_stmt, Visitor};
use ruff_python_ast::{self as ast, Pattern};
use ruff_python_ast::{BoolOp, Expr}; use ruff_python_ast::{BoolOp, Expr};
use crate::ast_node_ref::AstNodeRef; use crate::ast_node_ref::AstNodeRef;
use crate::module_name::ModuleName; use crate::module_name::ModuleName;
use crate::semantic_index::ast_ids::node_key::ExpressionNodeKey; use crate::semantic_index::ast_ids::node_key::ExpressionNodeKey;
use crate::semantic_index::ast_ids::AstIdsBuilder; use crate::semantic_index::ast_ids::AstIdsBuilder;
use crate::semantic_index::constraint::PatternConstraintKind;
use crate::semantic_index::definition::{ use crate::semantic_index::definition::{
AssignmentDefinitionNodeRef, ComprehensionDefinitionNodeRef, Definition, DefinitionNodeKey, AssignmentDefinitionNodeRef, ComprehensionDefinitionNodeRef, Definition, DefinitionNodeKey,
DefinitionNodeRef, ForStmtDefinitionNodeRef, ImportFromDefinitionNodeRef, DefinitionNodeRef, ForStmtDefinitionNodeRef, ImportFromDefinitionNodeRef,
@ -24,9 +25,12 @@ use crate::semantic_index::symbol::{
FileScopeId, NodeWithScopeKey, NodeWithScopeRef, Scope, ScopeId, ScopedSymbolId, FileScopeId, NodeWithScopeKey, NodeWithScopeRef, Scope, ScopeId, ScopedSymbolId,
SymbolTableBuilder, SymbolTableBuilder,
}; };
use crate::semantic_index::use_def::{FlowSnapshot, UseDefMapBuilder}; use crate::semantic_index::use_def::{
FlowSnapshot, ScopedConstraintId, ScopedVisibilityConstraintId, UseDefMapBuilder,
};
use crate::semantic_index::SemanticIndex; use crate::semantic_index::SemanticIndex;
use crate::unpack::Unpack; use crate::unpack::Unpack;
use crate::visibility_constraints::VisibilityConstraint;
use crate::Db; use crate::Db;
use super::constraint::{Constraint, ConstraintNode, PatternConstraint}; use super::constraint::{Constraint, ConstraintNode, PatternConstraint};
@ -285,10 +289,6 @@ impl<'db> SemanticIndexBuilder<'db> {
constraint constraint
} }
fn record_constraint(&mut self, constraint: Constraint<'db>) {
self.current_use_def_map_mut().record_constraint(constraint);
}
fn build_constraint(&mut self, constraint_node: &Expr) -> Constraint<'db> { fn build_constraint(&mut self, constraint_node: &Expr) -> Constraint<'db> {
let expression = self.add_standalone_expression(constraint_node); let expression = self.add_standalone_expression(constraint_node);
Constraint { Constraint {
@ -297,12 +297,89 @@ impl<'db> SemanticIndexBuilder<'db> {
} }
} }
fn record_negated_constraint(&mut self, constraint: Constraint<'db>) { /// Adds a new constraint to the list of all constraints, but does not record it. Returns the
/// constraint ID for later recording using [`SemanticIndexBuilder::record_constraint_id`].
fn add_constraint(&mut self, constraint: Constraint<'db>) -> ScopedConstraintId {
self.current_use_def_map_mut().add_constraint(constraint)
}
/// Negates a constraint and adds it to the list of all constraints, does not record it.
fn add_negated_constraint(
&mut self,
constraint: Constraint<'db>,
) -> (Constraint<'db>, ScopedConstraintId) {
let negated = Constraint {
node: constraint.node,
is_positive: false,
};
let id = self.current_use_def_map_mut().add_constraint(negated);
(negated, id)
}
/// Records a previously added constraint by adding it to all live bindings.
fn record_constraint_id(&mut self, constraint: ScopedConstraintId) {
self.current_use_def_map_mut() self.current_use_def_map_mut()
.record_constraint(Constraint { .record_constraint_id(constraint);
node: constraint.node, }
is_positive: false,
}); /// Adds and records a constraint, i.e. adds it to all live bindings.
fn record_constraint(&mut self, constraint: Constraint<'db>) {
self.current_use_def_map_mut().record_constraint(constraint);
}
/// Negates the given constraint and then adds it to all live bindings.
fn record_negated_constraint(&mut self, constraint: Constraint<'db>) -> ScopedConstraintId {
let (_, id) = self.add_negated_constraint(constraint);
self.record_constraint_id(id);
id
}
/// Adds a new visibility constraint, but does not record it. Returns the constraint ID
/// for later recording using [`SemanticIndexBuilder::record_visibility_constraint_id`].
fn add_visibility_constraint(
&mut self,
constraint: VisibilityConstraint<'db>,
) -> ScopedVisibilityConstraintId {
self.current_use_def_map_mut()
.add_visibility_constraint(constraint)
}
/// Records a previously added visibility constraint by applying it to all live bindings
/// and declarations.
fn record_visibility_constraint_id(&mut self, constraint: ScopedVisibilityConstraintId) {
self.current_use_def_map_mut()
.record_visibility_constraint_id(constraint);
}
/// Negates the given visibility constraint and then adds it to all live bindings and declarations.
fn record_negated_visibility_constraint(
&mut self,
constraint: ScopedVisibilityConstraintId,
) -> ScopedVisibilityConstraintId {
self.current_use_def_map_mut()
.record_visibility_constraint(VisibilityConstraint::VisibleIfNot(constraint))
}
/// Records a visibility constraint by applying it to all live bindings and declarations.
fn record_visibility_constraint(
&mut self,
constraint: Constraint<'db>,
) -> ScopedVisibilityConstraintId {
self.current_use_def_map_mut()
.record_visibility_constraint(VisibilityConstraint::VisibleIf(constraint))
}
/// Records a [`VisibilityConstraint::Ambiguous`] constraint.
fn record_ambiguous_visibility(&mut self) -> ScopedVisibilityConstraintId {
self.current_use_def_map_mut()
.record_visibility_constraint(VisibilityConstraint::Ambiguous)
}
/// Simplifies (resets) visibility constraints on all live bindings and declarations that did
/// not see any new definitions since the given snapshot.
fn simplify_visibility_constraints(&mut self, snapshot: FlowSnapshot) {
self.current_use_def_map_mut()
.simplify_visibility_constraints(snapshot);
} }
fn push_assignment(&mut self, assignment: CurrentAssignment<'db>) { fn push_assignment(&mut self, assignment: CurrentAssignment<'db>) {
@ -324,30 +401,37 @@ impl<'db> SemanticIndexBuilder<'db> {
fn add_pattern_constraint( fn add_pattern_constraint(
&mut self, &mut self,
subject: &ast::Expr, subject: Expression<'db>,
pattern: &ast::Pattern, pattern: &ast::Pattern,
) -> PatternConstraint<'db> { guard: Option<&ast::Expr>,
#[allow(unsafe_code)] ) -> Constraint<'db> {
let (subject, pattern) = unsafe { let guard = guard.map(|guard| self.add_standalone_expression(guard));
(
AstNodeRef::new(self.module.clone(), subject), let kind = match pattern {
AstNodeRef::new(self.module.clone(), pattern), Pattern::MatchValue(pattern) => {
) let value = self.add_standalone_expression(&pattern.value);
PatternConstraintKind::Value(value, guard)
}
Pattern::MatchSingleton(singleton) => {
PatternConstraintKind::Singleton(singleton.value, guard)
}
_ => PatternConstraintKind::Unsupported,
}; };
let pattern_constraint = PatternConstraint::new( let pattern_constraint = PatternConstraint::new(
self.db, self.db,
self.file, self.file,
self.current_scope(), self.current_scope(),
subject, subject,
pattern, kind,
countme::Count::default(), countme::Count::default(),
); );
self.current_use_def_map_mut() let constraint = Constraint {
.record_constraint(Constraint { node: ConstraintNode::Pattern(pattern_constraint),
node: ConstraintNode::Pattern(pattern_constraint), is_positive: true,
is_positive: true, };
}); self.current_use_def_map_mut().record_constraint(constraint);
pattern_constraint constraint
} }
/// Record an expression that needs to be a Salsa ingredient, because we need to infer its type /// Record an expression that needs to be a Salsa ingredient, because we need to infer its type
@ -799,6 +883,10 @@ where
let constraint = self.record_expression_constraint(&node.test); let constraint = self.record_expression_constraint(&node.test);
let mut constraints = vec![constraint]; let mut constraints = vec![constraint];
self.visit_body(&node.body); self.visit_body(&node.body);
let visibility_constraint_id = self.record_visibility_constraint(constraint);
let mut vis_constraints = vec![visibility_constraint_id];
let mut post_clauses: Vec<FlowSnapshot> = vec![]; let mut post_clauses: Vec<FlowSnapshot> = vec![];
let elif_else_clauses = node let elif_else_clauses = node
.elif_else_clauses .elif_else_clauses
@ -825,15 +913,31 @@ where
for constraint in &constraints { for constraint in &constraints {
self.record_negated_constraint(*constraint); self.record_negated_constraint(*constraint);
} }
if let Some(elif_test) = clause_test {
let elif_constraint = if let Some(elif_test) = clause_test {
self.visit_expr(elif_test); self.visit_expr(elif_test);
constraints.push(self.record_expression_constraint(elif_test)); let constraint = self.record_expression_constraint(elif_test);
} constraints.push(constraint);
Some(constraint)
} else {
None
};
self.visit_body(clause_body); self.visit_body(clause_body);
for id in &vis_constraints {
self.record_negated_visibility_constraint(*id);
}
if let Some(elif_constraint) = elif_constraint {
let id = self.record_visibility_constraint(elif_constraint);
vis_constraints.push(id);
}
} }
for post_clause_state in post_clauses { for post_clause_state in post_clauses {
self.flow_merge(post_clause_state); self.flow_merge(post_clause_state);
} }
self.simplify_visibility_constraints(pre_if);
} }
ast::Stmt::While(ast::StmtWhile { ast::Stmt::While(ast::StmtWhile {
test, test,
@ -856,6 +960,8 @@ where
self.visit_body(body); self.visit_body(body);
self.set_inside_loop(outer_loop_state); self.set_inside_loop(outer_loop_state);
let vis_constraint_id = self.record_visibility_constraint(constraint);
// Get the break states from the body of this loop, and restore the saved outer // Get the break states from the body of this loop, and restore the saved outer
// ones. // ones.
let break_states = let break_states =
@ -863,15 +969,21 @@ where
// We may execute the `else` clause without ever executing the body, so merge in // We may execute the `else` clause without ever executing the body, so merge in
// the pre-loop state before visiting `else`. // the pre-loop state before visiting `else`.
self.flow_merge(pre_loop); self.flow_merge(pre_loop.clone());
self.record_negated_constraint(constraint); self.record_negated_constraint(constraint);
self.visit_body(orelse); self.visit_body(orelse);
self.record_negated_visibility_constraint(vis_constraint_id);
// Breaking out of a while loop bypasses the `else` clause, so merge in the break // Breaking out of a while loop bypasses the `else` clause, so merge in the break
// states after visiting `else`. // states after visiting `else`.
for break_state in break_states { for break_state in break_states {
self.flow_merge(break_state); let snapshot = self.flow_snapshot();
self.flow_restore(break_state);
self.record_visibility_constraint(constraint);
self.flow_merge(snapshot);
} }
self.simplify_visibility_constraints(pre_loop);
} }
ast::Stmt::With(ast::StmtWith { ast::Stmt::With(ast::StmtWith {
items, items,
@ -912,6 +1024,8 @@ where
self.add_standalone_expression(iter); self.add_standalone_expression(iter);
self.visit_expr(iter); self.visit_expr(iter);
self.record_ambiguous_visibility();
let pre_loop = self.flow_snapshot(); let pre_loop = self.flow_snapshot();
let saved_break_states = std::mem::take(&mut self.loop_break_states); let saved_break_states = std::mem::take(&mut self.loop_break_states);
@ -947,32 +1061,63 @@ where
cases, cases,
range: _, range: _,
}) => { }) => {
self.add_standalone_expression(subject); let subject_expr = self.add_standalone_expression(subject);
self.visit_expr(subject); self.visit_expr(subject);
let after_subject = self.flow_snapshot(); let after_subject = self.flow_snapshot();
let Some((first, remaining)) = cases.split_first() else { let Some((first, remaining)) = cases.split_first() else {
return; return;
}; };
self.add_pattern_constraint(subject, &first.pattern);
let first_constraint_id = self.add_pattern_constraint(
subject_expr,
&first.pattern,
first.guard.as_deref(),
);
self.visit_match_case(first); self.visit_match_case(first);
let first_vis_constraint_id =
self.record_visibility_constraint(first_constraint_id);
let mut vis_constraints = vec![first_vis_constraint_id];
let mut post_case_snapshots = vec![]; let mut post_case_snapshots = vec![];
for case in remaining { for case in remaining {
post_case_snapshots.push(self.flow_snapshot()); post_case_snapshots.push(self.flow_snapshot());
self.flow_restore(after_subject.clone()); self.flow_restore(after_subject.clone());
self.add_pattern_constraint(subject, &case.pattern); let constraint_id = self.add_pattern_constraint(
subject_expr,
&case.pattern,
case.guard.as_deref(),
);
self.visit_match_case(case); self.visit_match_case(case);
for id in &vis_constraints {
self.record_negated_visibility_constraint(*id);
}
let vis_constraint_id = self.record_visibility_constraint(constraint_id);
vis_constraints.push(vis_constraint_id);
} }
for post_clause_state in post_case_snapshots {
self.flow_merge(post_clause_state); // If there is no final wildcard match case, pretend there is one. This is similar to how
} // we add an implicit `else` block in if-elif chains, in case it's not present.
if !cases if !cases
.last() .last()
.is_some_and(|case| case.guard.is_none() && case.pattern.is_wildcard()) .is_some_and(|case| case.guard.is_none() && case.pattern.is_wildcard())
{ {
self.flow_merge(after_subject); post_case_snapshots.push(self.flow_snapshot());
self.flow_restore(after_subject.clone());
for id in &vis_constraints {
self.record_negated_visibility_constraint(*id);
}
} }
for post_clause_state in post_case_snapshots {
self.flow_merge(post_clause_state);
}
self.simplify_visibility_constraints(after_subject);
} }
ast::Stmt::Try(ast::StmtTry { ast::Stmt::Try(ast::StmtTry {
body, body,
@ -982,6 +1127,8 @@ where
is_star, is_star,
range: _, range: _,
}) => { }) => {
self.record_ambiguous_visibility();
// Save the state prior to visiting any of the `try` block. // Save the state prior to visiting any of the `try` block.
// //
// Potentially none of the `try` block could have been executed prior to executing // Potentially none of the `try` block could have been executed prior to executing
@ -1222,19 +1369,19 @@ where
ast::Expr::If(ast::ExprIf { ast::Expr::If(ast::ExprIf {
body, test, orelse, .. body, test, orelse, ..
}) => { }) => {
// TODO detect statically known truthy or falsy test (via type inference, not naive
// AST inspection, so we can't simplify here, need to record test expression for
// later checking)
self.visit_expr(test); self.visit_expr(test);
let pre_if = self.flow_snapshot(); let pre_if = self.flow_snapshot();
let constraint = self.record_expression_constraint(test); let constraint = self.record_expression_constraint(test);
self.visit_expr(body); self.visit_expr(body);
let visibility_constraint = self.record_visibility_constraint(constraint);
let post_body = self.flow_snapshot(); let post_body = self.flow_snapshot();
self.flow_restore(pre_if); self.flow_restore(pre_if.clone());
self.record_negated_constraint(constraint); self.record_negated_constraint(constraint);
self.visit_expr(orelse); self.visit_expr(orelse);
self.record_negated_visibility_constraint(visibility_constraint);
self.flow_merge(post_body); self.flow_merge(post_body);
self.simplify_visibility_constraints(pre_if);
} }
ast::Expr::ListComp( ast::Expr::ListComp(
list_comprehension @ ast::ExprListComp { list_comprehension @ ast::ExprListComp {
@ -1291,27 +1438,55 @@ where
range: _, range: _,
op, op,
}) => { }) => {
// TODO detect statically known truthy or falsy values (via type inference, not naive let pre_op = self.flow_snapshot();
// AST inspection, so we can't simplify here, need to record test expression for
// later checking)
let mut snapshots = vec![]; let mut snapshots = vec![];
let mut visibility_constraints = vec![];
for (index, value) in values.iter().enumerate() { for (index, value) in values.iter().enumerate() {
self.visit_expr(value); self.visit_expr(value);
// In the last value we don't need to take a snapshot nor add a constraint
for vid in &visibility_constraints {
self.record_visibility_constraint_id(*vid);
}
// For the last value, we don't need to model control flow. There is short-circuiting
// anymore.
if index < values.len() - 1 { if index < values.len() - 1 {
// Snapshot is taken after visiting the expression but before adding the constraint.
snapshots.push(self.flow_snapshot());
let constraint = self.build_constraint(value); let constraint = self.build_constraint(value);
match op { let (constraint, constraint_id) = match op {
BoolOp::And => self.record_constraint(constraint), BoolOp::And => (constraint, self.add_constraint(constraint)),
BoolOp::Or => self.record_negated_constraint(constraint), BoolOp::Or => self.add_negated_constraint(constraint),
};
let visibility_constraint = self
.add_visibility_constraint(VisibilityConstraint::VisibleIf(constraint));
let after_expr = self.flow_snapshot();
// We first model the short-circuiting behavior. We take the short-circuit
// path here if all of the previous short-circuit paths were not taken, so
// we record all previously existing visibility constraints, and negate the
// one for the current expression.
for vid in &visibility_constraints {
self.record_visibility_constraint_id(*vid);
} }
self.record_negated_visibility_constraint(visibility_constraint);
snapshots.push(self.flow_snapshot());
// Then we model the non-short-circuiting behavior. Here, we need to delay
// the application of the visibility constraint until after the expression
// has been evaluated, so we only push it onto the stack here.
self.flow_restore(after_expr);
self.record_constraint_id(constraint_id);
visibility_constraints.push(visibility_constraint);
} }
} }
for snapshot in snapshots { for snapshot in snapshots {
self.flow_merge(snapshot); self.flow_merge(snapshot);
} }
self.simplify_visibility_constraints(pre_op);
} }
_ => { _ => {
walk_expr(self, expr); walk_expr(self, expr);

15
crates/red_knot_python_semantic/src/semantic_index/constraint.rs
View File

@ -1,7 +1,6 @@
use ruff_db::files::File; use ruff_db::files::File;
use ruff_python_ast as ast; use ruff_python_ast::Singleton;
use crate::ast_node_ref::AstNodeRef;
use crate::db::Db; use crate::db::Db;
use crate::semantic_index::expression::Expression; use crate::semantic_index::expression::Expression;
use crate::semantic_index::symbol::{FileScopeId, ScopeId}; use crate::semantic_index::symbol::{FileScopeId, ScopeId};
@ -18,6 +17,14 @@ pub(crate) enum ConstraintNode<'db> {
Pattern(PatternConstraint<'db>), Pattern(PatternConstraint<'db>),
} }
/// Pattern kinds for which we support type narrowing and/or static visibility analysis.
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum PatternConstraintKind<'db> {
Singleton(Singleton, Option<Expression<'db>>),
Value(Expression<'db>, Option<Expression<'db>>),
Unsupported,
}
#[salsa::tracked] #[salsa::tracked]
pub(crate) struct PatternConstraint<'db> { pub(crate) struct PatternConstraint<'db> {
#[id] #[id]
@ -28,11 +35,11 @@ pub(crate) struct PatternConstraint<'db> {
#[no_eq] #[no_eq]
#[return_ref] #[return_ref]
pub(crate) subject: AstNodeRef<ast::Expr>, pub(crate) subject: Expression<'db>,
#[no_eq] #[no_eq]
#[return_ref] #[return_ref]
pub(crate) pattern: AstNodeRef<ast::Pattern>, pub(crate) kind: PatternConstraintKind<'db>,
#[no_eq] #[no_eq]
count: countme::Count<PatternConstraint<'static>>, count: countme::Count<PatternConstraint<'static>>,

324
crates/red_knot_python_semantic/src/semantic_index/use_def.rs
View File

@ -169,17 +169,11 @@
//! indexvecs in the [`UseDefMap`]. //! indexvecs in the [`UseDefMap`].
//! //!
//! There is another special kind of possible "definition" for a symbol: there might be a path from //! There is another special kind of possible "definition" for a symbol: there might be a path from
//! the scope entry to a given use in which the symbol is never bound. //! the scope entry to a given use in which the symbol is never bound. We model this with a special
//! //! "unbound" definition (a `None` entry at the start of the `all_definitions` vector). If that
//! The simplest way to model "unbound" would be as a "binding" itself: the initial "binding" for //! sentinel definition is present in the live bindings at a given use, it means that there is a
//! each symbol in a scope. But actually modeling it this way would unnecessarily increase the //! possible path through control flow in which that symbol is unbound. Similarly, if that sentinel
//! number of [`Definition`]s that Salsa must track. Since "unbound" is special in that all symbols //! is present in the live declarations, it means that the symbol is (possibly) undeclared.
//! share it, and it doesn't have any additional per-symbol state, and constraints are irrelevant
//! to it, we can represent it more efficiently: we use the `may_be_unbound` boolean on the
//! [`SymbolBindings`] struct. If this flag is `true` for a use of a symbol, it means the symbol
//! has a path to the use in which it is never bound. If this flag is `false`, it means we've
//! eliminated the possibility of unbound: every control flow path to the use includes a binding
//! for this symbol.
//! //!
//! To build a [`UseDefMap`], the [`UseDefMapBuilder`] is notified of each new use, definition, and //! To build a [`UseDefMap`], the [`UseDefMapBuilder`] is notified of each new use, definition, and
//! constraint as they are encountered by the //! constraint as they are encountered by the
@ -190,11 +184,13 @@
//! end of the scope, it records the state for each symbol as the public definitions of that //! end of the scope, it records the state for each symbol as the public definitions of that
//! symbol. //! symbol.
//! //!
//! Let's walk through the above example. Initially we record for `x` that it has no bindings, and //! Let's walk through the above example. Initially we do not have any record of `x`. When we add
//! may be unbound. When we see `x = 1`, we record that as the sole live binding of `x`, and flip //! the new symbol (before we process the first binding), we create a new undefined `SymbolState`
//! `may_be_unbound` to `false`. Then we see `x = 2`, and we replace `x = 1` as the sole live //! which has a single live binding (the "unbound" definition) and a single live declaration (the
//! binding of `x`. When we get to `y = x`, we record that the live bindings for that use of `x` //! "undeclared" definition). When we see `x = 1`, we record that as the sole live binding of `x`.
//! are just the `x = 2` definition. //! The "unbound" binding is no longer visible. Then we see `x = 2`, and we replace `x = 1` as the
//! sole live binding of `x`. When we get to `y = x`, we record that the live bindings for that use
//! of `x` are just the `x = 2` definition.
//! //!
//! Then we hit the `if` branch. We visit the `test` node (`flag` in this case), since that will //! Then we hit the `if` branch. We visit the `test` node (`flag` in this case), since that will
//! happen regardless. Then we take a pre-branch snapshot of the current state for all symbols, //! happen regardless. Then we take a pre-branch snapshot of the current state for all symbols,
@ -207,8 +203,8 @@
//! be the pre-if conditions; if we are entering the `else` clause, we know that the `if` test //! be the pre-if conditions; if we are entering the `else` clause, we know that the `if` test
//! failed and we didn't execute the `if` body. So we first reset the builder to the pre-if state, //! failed and we didn't execute the `if` body. So we first reset the builder to the pre-if state,
//! using the snapshot we took previously (meaning we now have `x = 2` as the sole binding for `x` //! using the snapshot we took previously (meaning we now have `x = 2` as the sole binding for `x`
//! again), then visit the `else` clause, where `x = 4` replaces `x = 2` as the sole live binding //! again), and record a *negative* `flag` constraint for all live bindings (`x = 2`). We then
//! of `x`. //! visit the `else` clause, where `x = 4` replaces `x = 2` as the sole live binding of `x`.
//! //!
//! Now we reach the end of the if/else, and want to visit the following code. The state here needs //! Now we reach the end of the if/else, and want to visit the following code. The state here needs
//! to reflect that we might have gone through the `if` branch, or we might have gone through the //! to reflect that we might have gone through the `if` branch, or we might have gone through the
@ -217,18 +213,58 @@
//! snapshot (which has `x = 3` as the only live binding). The result of this merge is that we now //! snapshot (which has `x = 3` as the only live binding). The result of this merge is that we now
//! have two live bindings of `x`: `x = 3` and `x = 4`. //! have two live bindings of `x`: `x = 3` and `x = 4`.
//! //!
//! Another piece of information that the `UseDefMap` needs to provide are visibility constraints.
//! These are similar to the narrowing constraints, but apply to bindings and declarations within a
//! control flow path. Consider the following example:
//! ```py
//! x = 1
//! if test:
//! x = 2
//! y = "y"
//! ```
//! In principle, there are two possible control flow paths here. However, if we can statically
//! infer `test` to be always truthy or always falsy (that is, `__bool__` of `test` is of type
//! `Literal[True]` or `Literal[False]`), we can rule out one of the possible paths. To support
//! this feature, we record a visibility constraint of `test` to all live bindings and declarations
//! *after* visiting the body of the `if` statement. And we record a negative visibility constraint
//! `~test` to all live bindings/declarations in the (implicit) `else` branch. For the example
//! above, we would record the following visibility constraints (adding the implicit "unbound"
//! definitions for clarity):
//! ```py
//! x = <unbound> # not live, shadowed by `x = 1`
//! y = <unbound> # visibility constraint: ~test
//!
//! x = 1 # visibility constraint: ~test
//! if test:
//! x = 2 # visibility constraint: test
//! y = "y" # visibility constraint: test
//! ```
//! When we encounter a use of `x` after this `if` statement, we would record two live bindings: `x
//! = 1` with a constraint of `~test`, and `x = 2` with a constraint of `test`. In type inference,
//! when we iterate over all live bindings, we can evaluate these constraints to determine if a
//! particular binding is actually visible. For example, if `test` is always truthy, we only see
//! the `x = 2` binding. If `test` is always falsy, we only see the `x = 1` binding. And if the
//! `__bool__` method of `test` returns type `bool`, we can see both bindings.
//!
//! Note that we also record visibility constraints for the start of the scope. This is important
//! to determine if a symbol is definitely bound, possibly unbound, or definitely unbound. In the
//! example above, The `y = <unbound>` binding is constrained by `~test`, so `y` would only be
//! definitely-bound if `test` is always truthy.
//!
//! The [`UseDefMapBuilder`] itself just exposes methods for taking a snapshot, resetting to a //! The [`UseDefMapBuilder`] itself just exposes methods for taking a snapshot, resetting to a
//! snapshot, and merging a snapshot into the current state. The logic using these methods lives in //! snapshot, and merging a snapshot into the current state. The logic using these methods lives in
//! [`SemanticIndexBuilder`](crate::semantic_index::builder::SemanticIndexBuilder), e.g. where it //! [`SemanticIndexBuilder`](crate::semantic_index::builder::SemanticIndexBuilder), e.g. where it
//! visits a `StmtIf` node. //! visits a `StmtIf` node.
use self::symbol_state::{ use self::symbol_state::{
BindingIdWithConstraintsIterator, ConstraintIdIterator, DeclarationIdIterator, BindingIdWithConstraintsIterator, ConstraintIdIterator, DeclarationIdIterator,
ScopedConstraintId, ScopedDefinitionId, SymbolBindings, SymbolDeclarations, SymbolState, ScopedDefinitionId, SymbolBindings, SymbolDeclarations, SymbolState,
}; };
pub(crate) use self::symbol_state::{ScopedConstraintId, ScopedVisibilityConstraintId};
use crate::semantic_index::ast_ids::ScopedUseId; use crate::semantic_index::ast_ids::ScopedUseId;
use crate::semantic_index::definition::Definition; use crate::semantic_index::definition::Definition;
use crate::semantic_index::symbol::ScopedSymbolId; use crate::semantic_index::symbol::ScopedSymbolId;
use crate::symbol::Boundness; use crate::semantic_index::use_def::symbol_state::DeclarationIdWithConstraint;
use crate::visibility_constraints::{VisibilityConstraint, VisibilityConstraints};
use ruff_index::IndexVec; use ruff_index::IndexVec;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
@ -237,14 +273,20 @@ use super::constraint::Constraint;
mod bitset; mod bitset;
mod symbol_state; mod symbol_state;
type AllConstraints<'db> = IndexVec<ScopedConstraintId, Constraint<'db>>;
/// Applicable definitions and constraints for every use of a name. /// Applicable definitions and constraints for every use of a name.
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub(crate) struct UseDefMap<'db> { pub(crate) struct UseDefMap<'db> {
/// Array of [`Definition`] in this scope. /// Array of [`Definition`] in this scope. Only the first entry should be `None`;
all_definitions: IndexVec<ScopedDefinitionId, Definition<'db>>, /// this represents the implicit "unbound"/"undeclared" definition of every symbol.
all_definitions: IndexVec<ScopedDefinitionId, Option<Definition<'db>>>,
/// Array of [`Constraint`] in this scope. /// Array of [`Constraint`] in this scope.
all_constraints: IndexVec<ScopedConstraintId, Constraint<'db>>, all_constraints: AllConstraints<'db>,
/// Array of [`VisibilityConstraint`]s in this scope.
visibility_constraints: VisibilityConstraints<'db>,
/// [`SymbolBindings`] reaching a [`ScopedUseId`]. /// [`SymbolBindings`] reaching a [`ScopedUseId`].
bindings_by_use: IndexVec<ScopedUseId, SymbolBindings>, bindings_by_use: IndexVec<ScopedUseId, SymbolBindings>,
@ -275,14 +317,6 @@ impl<'db> UseDefMap<'db> {
self.bindings_iterator(&self.bindings_by_use[use_id]) self.bindings_iterator(&self.bindings_by_use[use_id])
} }
pub(crate) fn use_boundness(&self, use_id: ScopedUseId) -> Boundness {
if self.bindings_by_use[use_id].may_be_unbound() {
Boundness::PossiblyUnbound
} else {
Boundness::Bound
}
}
pub(crate) fn public_bindings( pub(crate) fn public_bindings(
&self, &self,
symbol: ScopedSymbolId, symbol: ScopedSymbolId,
@ -290,14 +324,6 @@ impl<'db> UseDefMap<'db> {
self.bindings_iterator(self.public_symbols[symbol].bindings()) self.bindings_iterator(self.public_symbols[symbol].bindings())
} }
pub(crate) fn public_boundness(&self, symbol: ScopedSymbolId) -> Boundness {
if self.public_symbols[symbol].may_be_unbound() {
Boundness::PossiblyUnbound
} else {
Boundness::Bound
}
}
pub(crate) fn bindings_at_declaration( pub(crate) fn bindings_at_declaration(
&self, &self,
declaration: Definition<'db>, declaration: Definition<'db>,
@ -310,10 +336,10 @@ impl<'db> UseDefMap<'db> {
} }
} }
pub(crate) fn declarations_at_binding( pub(crate) fn declarations_at_binding<'map>(
&self, &'map self,
binding: Definition<'db>, binding: Definition<'db>,
) -> DeclarationsIterator<'_, 'db> { ) -> DeclarationsIterator<'map, 'db> {
if let SymbolDefinitions::Declarations(declarations) = if let SymbolDefinitions::Declarations(declarations) =
&self.definitions_by_definition[&binding] &self.definitions_by_definition[&binding]
{ {
@ -323,37 +349,34 @@ impl<'db> UseDefMap<'db> {
} }
} }
pub(crate) fn public_declarations( pub(crate) fn public_declarations<'map>(
&self, &'map self,
symbol: ScopedSymbolId, symbol: ScopedSymbolId,
) -> DeclarationsIterator<'_, 'db> { ) -> DeclarationsIterator<'map, 'db> {
let declarations = self.public_symbols[symbol].declarations(); let declarations = self.public_symbols[symbol].declarations();
self.declarations_iterator(declarations) self.declarations_iterator(declarations)
} }
pub(crate) fn has_public_declarations(&self, symbol: ScopedSymbolId) -> bool { fn bindings_iterator<'map>(
!self.public_symbols[symbol].declarations().is_empty() &'map self,
} bindings: &'map SymbolBindings,
) -> BindingWithConstraintsIterator<'map, 'db> {
fn bindings_iterator<'a>(
&'a self,
bindings: &'a SymbolBindings,
) -> BindingWithConstraintsIterator<'a, 'db> {
BindingWithConstraintsIterator { BindingWithConstraintsIterator {
all_definitions: &self.all_definitions, all_definitions: &self.all_definitions,
all_constraints: &self.all_constraints, all_constraints: &self.all_constraints,
visibility_constraints: &self.visibility_constraints,
inner: bindings.iter(), inner: bindings.iter(),
} }
} }
fn declarations_iterator<'a>( fn declarations_iterator<'map>(
&'a self, &'map self,
declarations: &'a SymbolDeclarations, declarations: &'map SymbolDeclarations,
) -> DeclarationsIterator<'a, 'db> { ) -> DeclarationsIterator<'map, 'db> {
DeclarationsIterator { DeclarationsIterator {
all_definitions: &self.all_definitions, all_definitions: &self.all_definitions,
visibility_constraints: &self.visibility_constraints,
inner: declarations.iter(), inner: declarations.iter(),
may_be_undeclared: declarations.may_be_undeclared(),
} }
} }
} }
@ -367,8 +390,9 @@ enum SymbolDefinitions {
#[derive(Debug)] #[derive(Debug)]
pub(crate) struct BindingWithConstraintsIterator<'map, 'db> { pub(crate) struct BindingWithConstraintsIterator<'map, 'db> {
all_definitions: &'map IndexVec<ScopedDefinitionId, Definition<'db>>, all_definitions: &'map IndexVec<ScopedDefinitionId, Option<Definition<'db>>>,
all_constraints: &'map IndexVec<ScopedConstraintId, Constraint<'db>>, all_constraints: &'map AllConstraints<'db>,
pub(crate) visibility_constraints: &'map VisibilityConstraints<'db>,
inner: BindingIdWithConstraintsIterator<'map>, inner: BindingIdWithConstraintsIterator<'map>,
} }
@ -376,14 +400,17 @@ impl<'map, 'db> Iterator for BindingWithConstraintsIterator<'map, 'db> {
type Item = BindingWithConstraints<'map, 'db>; type Item = BindingWithConstraints<'map, 'db>;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
let all_constraints = self.all_constraints;
self.inner self.inner
.next() .next()
.map(|def_id_with_constraints| BindingWithConstraints { .map(|binding_id_with_constraints| BindingWithConstraints {
binding: self.all_definitions[def_id_with_constraints.definition], binding: self.all_definitions[binding_id_with_constraints.definition],
constraints: ConstraintsIterator { constraints: ConstraintsIterator {
all_constraints: self.all_constraints, all_constraints,
constraint_ids: def_id_with_constraints.constraint_ids, constraint_ids: binding_id_with_constraints.constraint_ids,
}, },
visibility_constraint: binding_id_with_constraints.visibility_constraint,
}) })
} }
} }
@ -391,12 +418,13 @@ impl<'map, 'db> Iterator for BindingWithConstraintsIterator<'map, 'db> {
impl std::iter::FusedIterator for BindingWithConstraintsIterator<'_, '_> {} impl std::iter::FusedIterator for BindingWithConstraintsIterator<'_, '_> {}
pub(crate) struct BindingWithConstraints<'map, 'db> { pub(crate) struct BindingWithConstraints<'map, 'db> {
pub(crate) binding: Definition<'db>, pub(crate) binding: Option<Definition<'db>>,
pub(crate) constraints: ConstraintsIterator<'map, 'db>, pub(crate) constraints: ConstraintsIterator<'map, 'db>,
pub(crate) visibility_constraint: ScopedVisibilityConstraintId,
} }
pub(crate) struct ConstraintsIterator<'map, 'db> { pub(crate) struct ConstraintsIterator<'map, 'db> {
all_constraints: &'map IndexVec<ScopedConstraintId, Constraint<'db>>, all_constraints: &'map AllConstraints<'db>,
constraint_ids: ConstraintIdIterator<'map>, constraint_ids: ConstraintIdIterator<'map>,
} }
@ -413,22 +441,31 @@ impl<'db> Iterator for ConstraintsIterator<'_, 'db> {
impl std::iter::FusedIterator for ConstraintsIterator<'_, '_> {} impl std::iter::FusedIterator for ConstraintsIterator<'_, '_> {}
pub(crate) struct DeclarationsIterator<'map, 'db> { pub(crate) struct DeclarationsIterator<'map, 'db> {
all_definitions: &'map IndexVec<ScopedDefinitionId, Definition<'db>>, all_definitions: &'map IndexVec<ScopedDefinitionId, Option<Definition<'db>>>,
pub(crate) visibility_constraints: &'map VisibilityConstraints<'db>,
inner: DeclarationIdIterator<'map>, inner: DeclarationIdIterator<'map>,
may_be_undeclared: bool,
} }
impl DeclarationsIterator<'_, '_> { pub(crate) struct DeclarationWithConstraint<'db> {
pub(crate) fn may_be_undeclared(&self) -> bool { pub(crate) declaration: Option<Definition<'db>>,
self.may_be_undeclared pub(crate) visibility_constraint: ScopedVisibilityConstraintId,
}
} }
impl<'db> Iterator for DeclarationsIterator<'_, 'db> { impl<'db> Iterator for DeclarationsIterator<'_, 'db> {
type Item = Definition<'db>; type Item = DeclarationWithConstraint<'db>;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
self.inner.next().map(|def_id| self.all_definitions[def_id]) self.inner.next().map(
|DeclarationIdWithConstraint {
definition,
visibility_constraint,
}| {
DeclarationWithConstraint {
declaration: self.all_definitions[definition],
visibility_constraint,
}
},
)
} }
} }
@ -438,15 +475,25 @@ impl std::iter::FusedIterator for DeclarationsIterator<'_, '_> {}
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub(super) struct FlowSnapshot { pub(super) struct FlowSnapshot {
symbol_states: IndexVec<ScopedSymbolId, SymbolState>, symbol_states: IndexVec<ScopedSymbolId, SymbolState>,
scope_start_visibility: ScopedVisibilityConstraintId,
} }
#[derive(Debug, Default)] #[derive(Debug)]
pub(super) struct UseDefMapBuilder<'db> { pub(super) struct UseDefMapBuilder<'db> {
/// Append-only array of [`Definition`]. /// Append-only array of [`Definition`].
all_definitions: IndexVec<ScopedDefinitionId, Definition<'db>>, all_definitions: IndexVec<ScopedDefinitionId, Option<Definition<'db>>>,
/// Append-only array of [`Constraint`]. /// Append-only array of [`Constraint`].
all_constraints: IndexVec<ScopedConstraintId, Constraint<'db>>, all_constraints: AllConstraints<'db>,
/// Append-only array of [`VisibilityConstraint`].
visibility_constraints: VisibilityConstraints<'db>,
/// A constraint which describes the visibility of the unbound/undeclared state, i.e.
/// whether or not the start of the scope is visible. This is important for cases like
/// `if True: x = 1; use(x)` where we need to hide the implicit "x = unbound" binding
/// in the "else" branch.
scope_start_visibility: ScopedVisibilityConstraintId,
/// Live bindings at each so-far-recorded use. /// Live bindings at each so-far-recorded use.
bindings_by_use: IndexVec<ScopedUseId, SymbolBindings>, bindings_by_use: IndexVec<ScopedUseId, SymbolBindings>,
@ -458,14 +505,30 @@ pub(super) struct UseDefMapBuilder<'db> {
symbol_states: IndexVec<ScopedSymbolId, SymbolState>, symbol_states: IndexVec<ScopedSymbolId, SymbolState>,
} }
impl Default for UseDefMapBuilder<'_> {
fn default() -> Self {
Self {
all_definitions: IndexVec::from_iter([None]),
all_constraints: IndexVec::new(),
visibility_constraints: VisibilityConstraints::default(),
scope_start_visibility: ScopedVisibilityConstraintId::ALWAYS_TRUE,
bindings_by_use: IndexVec::new(),
definitions_by_definition: FxHashMap::default(),
symbol_states: IndexVec::new(),
}
}
}
impl<'db> UseDefMapBuilder<'db> { impl<'db> UseDefMapBuilder<'db> {
pub(super) fn add_symbol(&mut self, symbol: ScopedSymbolId) { pub(super) fn add_symbol(&mut self, symbol: ScopedSymbolId) {
let new_symbol = self.symbol_states.push(SymbolState::undefined()); let new_symbol = self
.symbol_states
.push(SymbolState::undefined(self.scope_start_visibility));
debug_assert_eq!(symbol, new_symbol); debug_assert_eq!(symbol, new_symbol);
} }
pub(super) fn record_binding(&mut self, symbol: ScopedSymbolId, binding: Definition<'db>) { pub(super) fn record_binding(&mut self, symbol: ScopedSymbolId, binding: Definition<'db>) {
let def_id = self.all_definitions.push(binding); let def_id = self.all_definitions.push(Some(binding));
let symbol_state = &mut self.symbol_states[symbol]; let symbol_state = &mut self.symbol_states[symbol];
self.definitions_by_definition.insert( self.definitions_by_definition.insert(
binding, binding,
@ -474,10 +537,82 @@ impl<'db> UseDefMapBuilder<'db> {
symbol_state.record_binding(def_id); symbol_state.record_binding(def_id);
} }
pub(super) fn record_constraint(&mut self, constraint: Constraint<'db>) { pub(super) fn add_constraint(&mut self, constraint: Constraint<'db>) -> ScopedConstraintId {
let constraint_id = self.all_constraints.push(constraint); self.all_constraints.push(constraint)
}
pub(super) fn record_constraint_id(&mut self, constraint: ScopedConstraintId) {
for state in &mut self.symbol_states { for state in &mut self.symbol_states {
state.record_constraint(constraint_id); state.record_constraint(constraint);
}
}
pub(super) fn record_constraint(&mut self, constraint: Constraint<'db>) -> ScopedConstraintId {
let new_constraint_id = self.add_constraint(constraint);
self.record_constraint_id(new_constraint_id);
new_constraint_id
}
pub(super) fn add_visibility_constraint(
&mut self,
constraint: VisibilityConstraint<'db>,
) -> ScopedVisibilityConstraintId {
self.visibility_constraints.add(constraint)
}
pub(super) fn record_visibility_constraint_id(
&mut self,
constraint: ScopedVisibilityConstraintId,
) {
for state in &mut self.symbol_states {
state.record_visibility_constraint(&mut self.visibility_constraints, constraint);
}
self.scope_start_visibility = self
.visibility_constraints
.add_and_constraint(self.scope_start_visibility, constraint);
}
pub(super) fn record_visibility_constraint(
&mut self,
constraint: VisibilityConstraint<'db>,
) -> ScopedVisibilityConstraintId {
let new_constraint_id = self.add_visibility_constraint(constraint);
self.record_visibility_constraint_id(new_constraint_id);
new_constraint_id
}
/// This method resets the visibility constraints for all symbols to a previous state
/// *if* there have been no new declarations or bindings since then. Consider the
/// following example:
/// ```py
/// x = 0
/// y = 0
/// if test_a:
/// y = 1
/// elif test_b:
/// y = 2
/// elif test_c:
/// y = 3
///
/// # RESET
/// ```
/// We build a complex visibility constraint for the `y = 0` binding. We build the same
/// constraint for the `x = 0` binding as well, but at the `RESET` point, we can get rid
/// of it, as the `if`-`elif`-`elif` chain doesn't include any new bindings of `x`.
pub(super) fn simplify_visibility_constraints(&mut self, snapshot: FlowSnapshot) {
debug_assert!(self.symbol_states.len() >= snapshot.symbol_states.len());
self.scope_start_visibility = snapshot.scope_start_visibility;
// Note that this loop terminates when we reach a symbol not present in the snapshot.
// This means we keep visibility constraints for all new symbols, which is intended,
// since these symbols have been introduced in the corresponding branch, which might
// be subject to visibility constraints. We only simplify/reset visibility constraints
// for symbols that have the same bindings and declarations present compared to the
// snapshot.
for (current, snapshot) in self.symbol_states.iter_mut().zip(snapshot.symbol_states) {
current.simplify_visibility_constraints(snapshot);
} }
} }
@ -486,7 +621,7 @@ impl<'db> UseDefMapBuilder<'db> {
symbol: ScopedSymbolId, symbol: ScopedSymbolId,
declaration: Definition<'db>, declaration: Definition<'db>,
) { ) {
let def_id = self.all_definitions.push(declaration); let def_id = self.all_definitions.push(Some(declaration));
let symbol_state = &mut self.symbol_states[symbol]; let symbol_state = &mut self.symbol_states[symbol];
self.definitions_by_definition.insert( self.definitions_by_definition.insert(
declaration, declaration,
@ -501,7 +636,7 @@ impl<'db> UseDefMapBuilder<'db> {
definition: Definition<'db>, definition: Definition<'db>,
) { ) {
// We don't need to store anything in self.definitions_by_definition. // We don't need to store anything in self.definitions_by_definition.
let def_id = self.all_definitions.push(definition); let def_id = self.all_definitions.push(Some(definition));
let symbol_state = &mut self.symbol_states[symbol]; let symbol_state = &mut self.symbol_states[symbol];
symbol_state.record_declaration(def_id); symbol_state.record_declaration(def_id);
symbol_state.record_binding(def_id); symbol_state.record_binding(def_id);
@ -520,6 +655,7 @@ impl<'db> UseDefMapBuilder<'db> {
pub(super) fn snapshot(&self) -> FlowSnapshot { pub(super) fn snapshot(&self) -> FlowSnapshot {
FlowSnapshot { FlowSnapshot {
symbol_states: self.symbol_states.clone(), symbol_states: self.symbol_states.clone(),
scope_start_visibility: self.scope_start_visibility,
} }
} }
@ -533,12 +669,15 @@ impl<'db> UseDefMapBuilder<'db> {
// Restore the current visible-definitions state to the given snapshot. // Restore the current visible-definitions state to the given snapshot.
self.symbol_states = snapshot.symbol_states; self.symbol_states = snapshot.symbol_states;
self.scope_start_visibility = snapshot.scope_start_visibility;
// If the snapshot we are restoring is missing some symbols we've recorded since, we need // If the snapshot we are restoring is missing some symbols we've recorded since, we need
// to fill them in so the symbol IDs continue to line up. Since they don't exist in the // to fill them in so the symbol IDs continue to line up. Since they don't exist in the
// snapshot, the correct state to fill them in with is "undefined". // snapshot, the correct state to fill them in with is "undefined".
self.symbol_states self.symbol_states.resize(
.resize(num_symbols, SymbolState::undefined()); num_symbols,
SymbolState::undefined(self.scope_start_visibility),
);
} }
/// Merge the given snapshot into the current state, reflecting that we might have taken either /// Merge the given snapshot into the current state, reflecting that we might have taken either
@ -553,13 +692,19 @@ impl<'db> UseDefMapBuilder<'db> {
let mut snapshot_definitions_iter = snapshot.symbol_states.into_iter(); let mut snapshot_definitions_iter = snapshot.symbol_states.into_iter();
for current in &mut self.symbol_states { for current in &mut self.symbol_states {
if let Some(snapshot) = snapshot_definitions_iter.next() { if let Some(snapshot) = snapshot_definitions_iter.next() {
current.merge(snapshot); current.merge(snapshot, &mut self.visibility_constraints);
} else { } else {
current.merge(
SymbolState::undefined(snapshot.scope_start_visibility),
&mut self.visibility_constraints,
);
// Symbol not present in snapshot, so it's unbound/undeclared from that path. // Symbol not present in snapshot, so it's unbound/undeclared from that path.
current.set_may_be_unbound();
current.set_may_be_undeclared();
} }
} }
self.scope_start_visibility = self
.visibility_constraints
.add_or_constraint(self.scope_start_visibility, snapshot.scope_start_visibility);
} }
pub(super) fn finish(mut self) -> UseDefMap<'db> { pub(super) fn finish(mut self) -> UseDefMap<'db> {
@ -572,6 +717,7 @@ impl<'db> UseDefMapBuilder<'db> {
UseDefMap { UseDefMap {
all_definitions: self.all_definitions, all_definitions: self.all_definitions,
all_constraints: self.all_constraints, all_constraints: self.all_constraints,
visibility_constraints: self.visibility_constraints,
bindings_by_use: self.bindings_by_use, bindings_by_use: self.bindings_by_use,
public_symbols: self.symbol_states, public_symbols: self.symbol_states,
definitions_by_definition: self.definitions_by_definition, definitions_by_definition: self.definitions_by_definition,

77
crates/red_knot_python_semantic/src/semantic_index/use_def/bitset.rs
View File

@ -32,10 +32,6 @@ impl<const B: usize> BitSet<B> {
bitset bitset
} }
pub(super) fn is_empty(&self) -> bool {
self.blocks().iter().all(|&b| b == 0)
}
/// Convert from Inline to Heap, if needed, and resize the Heap vector, if needed. /// Convert from Inline to Heap, if needed, and resize the Heap vector, if needed.
fn resize(&mut self, value: u32) { fn resize(&mut self, value: u32) {
let num_blocks_needed = (value / 64) + 1; let num_blocks_needed = (value / 64) + 1;
@ -97,19 +93,6 @@ impl<const B: usize> BitSet<B> {
} }
} }
/// Union in-place with another [`BitSet`].
pub(super) fn union(&mut self, other: &BitSet<B>) {
let mut max_len = self.blocks().len();
let other_len = other.blocks().len();
if other_len > max_len {
max_len = other_len;
self.resize_blocks(max_len);
}
for (my_block, other_block) in self.blocks_mut().iter_mut().zip(other.blocks()) {
*my_block |= other_block;
}
}
/// Return an iterator over the values (in ascending order) in this [`BitSet`]. /// Return an iterator over the values (in ascending order) in this [`BitSet`].
pub(super) fn iter(&self) -> BitSetIterator<'_, B> { pub(super) fn iter(&self) -> BitSetIterator<'_, B> {
let blocks = self.blocks(); let blocks = self.blocks();
@ -239,59 +222,6 @@ mod tests {
assert_bitset(&b1, &[89]); assert_bitset(&b1, &[89]);
} }
#[test]
fn union() {
let mut b1 = BitSet::<1>::with(2);
let b2 = BitSet::<1>::with(4);
b1.union(&b2);
assert_bitset(&b1, &[2, 4]);
}
#[test]
fn union_mixed_1() {
let mut b1 = BitSet::<1>::with(4);
let mut b2 = BitSet::<1>::with(4);
b1.insert(89);
b2.insert(5);
b1.union(&b2);
assert_bitset(&b1, &[4, 5, 89]);
}
#[test]
fn union_mixed_2() {
let mut b1 = BitSet::<1>::with(4);
let mut b2 = BitSet::<1>::with(4);
b1.insert(23);
b2.insert(89);
b1.union(&b2);
assert_bitset(&b1, &[4, 23, 89]);
}
#[test]
fn union_heap() {
let mut b1 = BitSet::<1>::with(4);
let mut b2 = BitSet::<1>::with(4);
b1.insert(89);
b2.insert(90);
b1.union(&b2);
assert_bitset(&b1, &[4, 89, 90]);
}
#[test]
fn union_heap_2() {
let mut b1 = BitSet::<1>::with(89);
let mut b2 = BitSet::<1>::with(89);
b1.insert(91);
b2.insert(90);
b1.union(&b2);
assert_bitset(&b1, &[89, 90, 91]);
}
#[test] #[test]
fn multiple_blocks() { fn multiple_blocks() {
let mut b = BitSet::<2>::with(120); let mut b = BitSet::<2>::with(120);
@ -299,11 +229,4 @@ mod tests {
assert!(matches!(b, BitSet::Inline(_))); assert!(matches!(b, BitSet::Inline(_)));
assert_bitset(&b, &[45, 120]); assert_bitset(&b, &[45, 120]);
} }
#[test]
fn empty() {
let b = BitSet::<1>::default();
assert!(b.is_empty());
}
} }

596
crates/red_knot_python_semantic/src/semantic_index/use_def/symbol_state.rs
View File

@ -43,6 +43,8 @@
//! //!
//! Tracking live declarations is simpler, since constraints are not involved, but otherwise very //! Tracking live declarations is simpler, since constraints are not involved, but otherwise very
//! similar to tracking live bindings. //! similar to tracking live bindings.
use crate::semantic_index::use_def::VisibilityConstraints;
use super::bitset::{BitSet, BitSetIterator}; use super::bitset::{BitSet, BitSetIterator};
use ruff_index::newtype_index; use ruff_index::newtype_index;
use smallvec::SmallVec; use smallvec::SmallVec;
@ -51,9 +53,18 @@ use smallvec::SmallVec;
#[newtype_index] #[newtype_index]
pub(super) struct ScopedDefinitionId; pub(super) struct ScopedDefinitionId;
impl ScopedDefinitionId {
/// A special ID that is used to describe an implicit start-of-scope state. When
/// we see that this definition is live, we know that the symbol is (possibly)
/// unbound or undeclared at a given usage site.
/// When creating a use-def-map builder, we always add an empty `None` definition
/// at index 0, so this ID is always present.
pub(super) const UNBOUND: ScopedDefinitionId = ScopedDefinitionId::from_u32(0);
}
/// A newtype-index for a constraint expression in a particular scope. /// A newtype-index for a constraint expression in a particular scope.
#[newtype_index] #[newtype_index]
pub(super) struct ScopedConstraintId; pub(crate) struct ScopedConstraintId;
/// Can reference this * 64 total definitions inline; more will fall back to the heap. /// Can reference this * 64 total definitions inline; more will fall back to the heap.
const INLINE_BINDING_BLOCKS: usize = 3; const INLINE_BINDING_BLOCKS: usize = 3;
@ -75,58 +86,97 @@ const INLINE_CONSTRAINT_BLOCKS: usize = 2;
/// Can keep inline this many live bindings per symbol at a given time; more will go to heap. /// Can keep inline this many live bindings per symbol at a given time; more will go to heap.
const INLINE_BINDINGS_PER_SYMBOL: usize = 4; const INLINE_BINDINGS_PER_SYMBOL: usize = 4;
/// One [`BitSet`] of applicable [`ScopedConstraintId`] per live binding. /// Which constraints apply to a given binding?
type InlineConstraintArray = [BitSet<INLINE_CONSTRAINT_BLOCKS>; INLINE_BINDINGS_PER_SYMBOL]; type Constraints = BitSet<INLINE_CONSTRAINT_BLOCKS>;
type Constraints = SmallVec<InlineConstraintArray>;
type ConstraintsIterator<'a> = std::slice::Iter<'a, BitSet<INLINE_CONSTRAINT_BLOCKS>>; type InlineConstraintArray = [Constraints; INLINE_BINDINGS_PER_SYMBOL];
/// One [`BitSet`] of applicable [`ScopedConstraintId`]s per live binding.
type ConstraintsPerBinding = SmallVec<InlineConstraintArray>;
/// Iterate over all constraints for a single binding.
type ConstraintsIterator<'a> = std::slice::Iter<'a, Constraints>;
type ConstraintsIntoIterator = smallvec::IntoIter<InlineConstraintArray>; type ConstraintsIntoIterator = smallvec::IntoIter<InlineConstraintArray>;
/// Live declarations for a single symbol at some point in control flow. /// A newtype-index for a visibility constraint in a particular scope.
#[newtype_index]
pub(crate) struct ScopedVisibilityConstraintId;
impl ScopedVisibilityConstraintId {
/// A special ID that is used for an "always true" / "always visible" constraint.
/// When we create a new [`VisibilityConstraints`] object, this constraint is always
/// present at index 0.
pub(crate) const ALWAYS_TRUE: ScopedVisibilityConstraintId =
ScopedVisibilityConstraintId::from_u32(0);
}
const INLINE_VISIBILITY_CONSTRAINTS: usize = 4;
type InlineVisibilityConstraintsArray =
[ScopedVisibilityConstraintId; INLINE_VISIBILITY_CONSTRAINTS];
/// One [`ScopedVisibilityConstraintId`] per live declaration.
type VisibilityConstraintPerDeclaration = SmallVec<InlineVisibilityConstraintsArray>;
/// One [`ScopedVisibilityConstraintId`] per live binding.
type VisibilityConstraintPerBinding = SmallVec<InlineVisibilityConstraintsArray>;
/// Iterator over the visibility constraints for all live bindings/declarations.
type VisibilityConstraintsIterator<'a> = std::slice::Iter<'a, ScopedVisibilityConstraintId>;
type VisibilityConstraintsIntoIterator = smallvec::IntoIter<InlineVisibilityConstraintsArray>;
/// Live declarations for a single symbol at some point in control flow, with their
/// corresponding visibility constraints.
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct SymbolDeclarations { pub(super) struct SymbolDeclarations {
/// [`BitSet`]: which declarations (as [`ScopedDefinitionId`]) can reach the current location? /// [`BitSet`]: which declarations (as [`ScopedDefinitionId`]) can reach the current location?
live_declarations: Declarations, pub(crate) live_declarations: Declarations,
/// Could the symbol be un-declared at this point? /// For each live declaration, which visibility constraint applies to it?
may_be_undeclared: bool, pub(crate) visibility_constraints: VisibilityConstraintPerDeclaration,
} }
impl SymbolDeclarations { impl SymbolDeclarations {
fn undeclared() -> Self { fn undeclared(scope_start_visibility: ScopedVisibilityConstraintId) -> Self {
Self { Self {
live_declarations: Declarations::default(), live_declarations: Declarations::with(0),
may_be_undeclared: true, visibility_constraints: VisibilityConstraintPerDeclaration::from_iter([
scope_start_visibility,
]),
} }
} }
/// Record a newly-encountered declaration for this symbol. /// Record a newly-encountered declaration for this symbol.
fn record_declaration(&mut self, declaration_id: ScopedDefinitionId) { fn record_declaration(&mut self, declaration_id: ScopedDefinitionId) {
self.live_declarations = Declarations::with(declaration_id.into()); self.live_declarations = Declarations::with(declaration_id.into());
self.may_be_undeclared = false;
self.visibility_constraints = VisibilityConstraintPerDeclaration::with_capacity(1);
self.visibility_constraints
.push(ScopedVisibilityConstraintId::ALWAYS_TRUE);
} }
/// Add undeclared as a possibility for this symbol. /// Add given visibility constraint to all live declarations.
fn set_may_be_undeclared(&mut self) { pub(super) fn record_visibility_constraint(
self.may_be_undeclared = true; &mut self,
visibility_constraints: &mut VisibilityConstraints,
constraint: ScopedVisibilityConstraintId,
) {
for existing in &mut self.visibility_constraints {
*existing = visibility_constraints.add_and_constraint(*existing, constraint);
}
} }
/// Return an iterator over live declarations for this symbol. /// Return an iterator over live declarations for this symbol.
pub(super) fn iter(&self) -> DeclarationIdIterator { pub(super) fn iter(&self) -> DeclarationIdIterator {
DeclarationIdIterator { DeclarationIdIterator {
inner: self.live_declarations.iter(), declarations: self.live_declarations.iter(),
visibility_constraints: self.visibility_constraints.iter(),
} }
} }
pub(super) fn is_empty(&self) -> bool {
self.live_declarations.is_empty()
}
pub(super) fn may_be_undeclared(&self) -> bool {
self.may_be_undeclared
}
} }
/// Live bindings and narrowing constraints for a single symbol at some point in control flow. /// Live bindings for a single symbol at some point in control flow. Each live binding comes
/// with a set of narrowing constraints and a visibility constraint.
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct SymbolBindings { pub(super) struct SymbolBindings {
/// [`BitSet`]: which bindings (as [`ScopedDefinitionId`]) can reach the current location? /// [`BitSet`]: which bindings (as [`ScopedDefinitionId`]) can reach the current location?
@ -136,34 +186,34 @@ pub(super) struct SymbolBindings {
/// ///
/// This is a [`smallvec::SmallVec`] which should always have one [`BitSet`] of constraints per /// This is a [`smallvec::SmallVec`] which should always have one [`BitSet`] of constraints per
/// binding in `live_bindings`. /// binding in `live_bindings`.
constraints: Constraints, constraints: ConstraintsPerBinding,
/// Could the symbol be unbound at this point? /// For each live binding, which visibility constraint applies to it?
may_be_unbound: bool, visibility_constraints: VisibilityConstraintPerBinding,
} }
impl SymbolBindings { impl SymbolBindings {
fn unbound() -> Self { fn unbound(scope_start_visibility: ScopedVisibilityConstraintId) -> Self {
Self { Self {
live_bindings: Bindings::default(), live_bindings: Bindings::with(ScopedDefinitionId::UNBOUND.as_u32()),
constraints: Constraints::default(), constraints: ConstraintsPerBinding::from_iter([Constraints::default()]),
may_be_unbound: true, visibility_constraints: VisibilityConstraintPerBinding::from_iter([
scope_start_visibility,
]),
} }
} }
/// Add Unbound as a possibility for this symbol.
fn set_may_be_unbound(&mut self) {
self.may_be_unbound = true;
}
/// Record a newly-encountered binding for this symbol. /// Record a newly-encountered binding for this symbol.
pub(super) fn record_binding(&mut self, binding_id: ScopedDefinitionId) { pub(super) fn record_binding(&mut self, binding_id: ScopedDefinitionId) {
// The new binding replaces all previous live bindings in this path, and has no // The new binding replaces all previous live bindings in this path, and has no
// constraints. // constraints.
self.live_bindings = Bindings::with(binding_id.into()); self.live_bindings = Bindings::with(binding_id.into());
self.constraints = Constraints::with_capacity(1); self.constraints = ConstraintsPerBinding::with_capacity(1);
self.constraints.push(BitSet::default()); self.constraints.push(Constraints::default());
self.may_be_unbound = false;
self.visibility_constraints = VisibilityConstraintPerBinding::with_capacity(1);
self.visibility_constraints
.push(ScopedVisibilityConstraintId::ALWAYS_TRUE);
} }
/// Add given constraint to all live bindings. /// Add given constraint to all live bindings.
@ -173,17 +223,25 @@ impl SymbolBindings {
} }
} }
/// Iterate over currently live bindings for this symbol. /// Add given visibility constraint to all live bindings.
pub(super) fn record_visibility_constraint(
&mut self,
visibility_constraints: &mut VisibilityConstraints,
constraint: ScopedVisibilityConstraintId,
) {
for existing in &mut self.visibility_constraints {
*existing = visibility_constraints.add_and_constraint(*existing, constraint);
}
}
/// Iterate over currently live bindings for this symbol
pub(super) fn iter(&self) -> BindingIdWithConstraintsIterator { pub(super) fn iter(&self) -> BindingIdWithConstraintsIterator {
BindingIdWithConstraintsIterator { BindingIdWithConstraintsIterator {
definitions: self.live_bindings.iter(), definitions: self.live_bindings.iter(),
constraints: self.constraints.iter(), constraints: self.constraints.iter(),
visibility_constraints: self.visibility_constraints.iter(),
} }
} }
pub(super) fn may_be_unbound(&self) -> bool {
self.may_be_unbound
}
} }
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
@ -194,20 +252,16 @@ pub(super) struct SymbolState {
impl SymbolState { impl SymbolState {
/// Return a new [`SymbolState`] representing an unbound, undeclared symbol. /// Return a new [`SymbolState`] representing an unbound, undeclared symbol.
pub(super) fn undefined() -> Self { pub(super) fn undefined(scope_start_visibility: ScopedVisibilityConstraintId) -> Self {
Self { Self {
declarations: SymbolDeclarations::undeclared(), declarations: SymbolDeclarations::undeclared(scope_start_visibility),
bindings: SymbolBindings::unbound(), bindings: SymbolBindings::unbound(scope_start_visibility),
} }
} }
/// Add Unbound as a possibility for this symbol.
pub(super) fn set_may_be_unbound(&mut self) {
self.bindings.set_may_be_unbound();
}
/// Record a newly-encountered binding for this symbol. /// Record a newly-encountered binding for this symbol.
pub(super) fn record_binding(&mut self, binding_id: ScopedDefinitionId) { pub(super) fn record_binding(&mut self, binding_id: ScopedDefinitionId) {
debug_assert_ne!(binding_id, ScopedDefinitionId::UNBOUND);
self.bindings.record_binding(binding_id); self.bindings.record_binding(binding_id);
} }
@ -216,9 +270,26 @@ impl SymbolState {
self.bindings.record_constraint(constraint_id); self.bindings.record_constraint(constraint_id);
} }
/// Add undeclared as a possibility for this symbol. /// Add given visibility constraint to all live bindings.
pub(super) fn set_may_be_undeclared(&mut self) { pub(super) fn record_visibility_constraint(
self.declarations.set_may_be_undeclared(); &mut self,
visibility_constraints: &mut VisibilityConstraints,
constraint: ScopedVisibilityConstraintId,
) {
self.bindings
.record_visibility_constraint(visibility_constraints, constraint);
self.declarations
.record_visibility_constraint(visibility_constraints, constraint);
}
pub(super) fn simplify_visibility_constraints(&mut self, snapshot_state: SymbolState) {
if self.bindings.live_bindings == snapshot_state.bindings.live_bindings {
self.bindings.visibility_constraints = snapshot_state.bindings.visibility_constraints;
}
if self.declarations.live_declarations == snapshot_state.declarations.live_declarations {
self.declarations.visibility_constraints =
snapshot_state.declarations.visibility_constraints;
}
} }
/// Record a newly-encountered declaration of this symbol. /// Record a newly-encountered declaration of this symbol.
@ -227,29 +298,31 @@ impl SymbolState {
} }
/// Merge another [`SymbolState`] into this one. /// Merge another [`SymbolState`] into this one.
pub(super) fn merge(&mut self, b: SymbolState) { pub(super) fn merge(
&mut self,
b: SymbolState,
visibility_constraints: &mut VisibilityConstraints,
) {
let mut a = Self { let mut a = Self {
bindings: SymbolBindings { bindings: SymbolBindings {
live_bindings: Bindings::default(), live_bindings: Bindings::default(),
constraints: Constraints::default(), constraints: ConstraintsPerBinding::default(),
may_be_unbound: self.bindings.may_be_unbound || b.bindings.may_be_unbound, visibility_constraints: VisibilityConstraintPerBinding::default(),
}, },
declarations: SymbolDeclarations { declarations: SymbolDeclarations {
live_declarations: self.declarations.live_declarations.clone(), live_declarations: self.declarations.live_declarations.clone(),
may_be_undeclared: self.declarations.may_be_undeclared visibility_constraints: VisibilityConstraintPerDeclaration::default(),
|| b.declarations.may_be_undeclared,
}, },
}; };
std::mem::swap(&mut a, self); std::mem::swap(&mut a, self);
self.declarations
.live_declarations
.union(&b.declarations.live_declarations);
let mut a_defs_iter = a.bindings.live_bindings.iter(); let mut a_defs_iter = a.bindings.live_bindings.iter();
let mut b_defs_iter = b.bindings.live_bindings.iter(); let mut b_defs_iter = b.bindings.live_bindings.iter();
let mut a_constraints_iter = a.bindings.constraints.into_iter(); let mut a_constraints_iter = a.bindings.constraints.into_iter();
let mut b_constraints_iter = b.bindings.constraints.into_iter(); let mut b_constraints_iter = b.bindings.constraints.into_iter();
let mut a_vis_constraints_iter = a.bindings.visibility_constraints.into_iter();
let mut b_vis_constraints_iter = b.bindings.visibility_constraints.into_iter();
let mut opt_a_def: Option<u32> = a_defs_iter.next(); let mut opt_a_def: Option<u32> = a_defs_iter.next();
let mut opt_b_def: Option<u32> = b_defs_iter.next(); let mut opt_b_def: Option<u32> = b_defs_iter.next();
@ -261,17 +334,30 @@ impl SymbolState {
// path is irrelevant. // path is irrelevant.
// Helper to push `def`, with constraints in `constraints_iter`, onto `self`. // Helper to push `def`, with constraints in `constraints_iter`, onto `self`.
let push = |def, constraints_iter: &mut ConstraintsIntoIterator, merged: &mut Self| { let push = |def,
constraints_iter: &mut ConstraintsIntoIterator,
visibility_constraints_iter: &mut VisibilityConstraintsIntoIterator,
merged: &mut Self| {
merged.bindings.live_bindings.insert(def); merged.bindings.live_bindings.insert(def);
// SAFETY: we only ever create SymbolState with either no definitions and no constraint // SAFETY: we only ever create SymbolState using [`SymbolState::undefined`], which adds
// bitsets (`::unbound`) or one definition and one constraint bitset (`::with`), and // one "unbound" definition with corresponding narrowing and visibility constraints, or
// `::merge` always pushes one definition and one constraint bitset together (just // using [`SymbolState::record_binding`] or [`SymbolState::record_declaration`], which
// below), so the number of definitions and the number of constraint bitsets can never // similarly add one definition with corresponding constraints. [`SymbolState::merge`]
// always pushes one definition and one constraint bitset and one visibility constraint
// together (just below), so the number of definitions and the number of constraints can
// never get out of sync.
// get out of sync. // get out of sync.
let constraints = constraints_iter let constraints = constraints_iter
.next() .next()
.expect("definitions and constraints length mismatch"); .expect("definitions and constraints length mismatch");
let visibility_constraints = visibility_constraints_iter
.next()
.expect("definitions and visibility_constraints length mismatch");
merged.bindings.constraints.push(constraints); merged.bindings.constraints.push(constraints);
merged
.bindings
.visibility_constraints
.push(visibility_constraints);
}; };
loop { loop {
@ -279,50 +365,139 @@ impl SymbolState {
(Some(a_def), Some(b_def)) => match a_def.cmp(&b_def) { (Some(a_def), Some(b_def)) => match a_def.cmp(&b_def) {
std::cmp::Ordering::Less => { std::cmp::Ordering::Less => {
// Next definition ID is only in `a`, push it to `self` and advance `a`. // Next definition ID is only in `a`, push it to `self` and advance `a`.
push(a_def, &mut a_constraints_iter, self); push(
a_def,
&mut a_constraints_iter,
&mut a_vis_constraints_iter,
self,
);
opt_a_def = a_defs_iter.next(); opt_a_def = a_defs_iter.next();
} }
std::cmp::Ordering::Greater => { std::cmp::Ordering::Greater => {
// Next definition ID is only in `b`, push it to `self` and advance `b`. // Next definition ID is only in `b`, push it to `self` and advance `b`.
push(b_def, &mut b_constraints_iter, self); push(
b_def,
&mut b_constraints_iter,
&mut b_vis_constraints_iter,
self,
);
opt_b_def = b_defs_iter.next(); opt_b_def = b_defs_iter.next();
} }
std::cmp::Ordering::Equal => { std::cmp::Ordering::Equal => {
// Next definition is in both; push to `self` and intersect constraints. // Next definition is in both; push to `self` and intersect constraints.
push(a_def, &mut b_constraints_iter, self); push(
// SAFETY: we only ever create SymbolState with either no definitions and a_def,
// no constraint bitsets (`::unbound`) or one definition and one constraint &mut b_constraints_iter,
// bitset (`::with`), and `::merge` always pushes one definition and one &mut b_vis_constraints_iter,
// constraint bitset together (just below), so the number of definitions self,
// and the number of constraint bitsets can never get out of sync. );
// SAFETY: see comment in `push` above.
let a_constraints = a_constraints_iter let a_constraints = a_constraints_iter
.next() .next()
.expect("definitions and constraints length mismatch"); .expect("definitions and constraints length mismatch");
let current_constraints = self.bindings.constraints.last_mut().unwrap();
// If the same definition is visible through both paths, any constraint // If the same definition is visible through both paths, any constraint
// that applies on only one path is irrelevant to the resulting type from // that applies on only one path is irrelevant to the resulting type from
// unioning the two paths, so we intersect the constraints. // unioning the two paths, so we intersect the constraints.
self.bindings current_constraints.intersect(&a_constraints);
.constraints
.last_mut() // For visibility constraints, we merge them using a ternary OR operation:
.unwrap() let a_vis_constraint = a_vis_constraints_iter
.intersect(&a_constraints); .next()
.expect("visibility_constraints length mismatch");
let current_vis_constraint =
self.bindings.visibility_constraints.last_mut().unwrap();
*current_vis_constraint = visibility_constraints
.add_or_constraint(*current_vis_constraint, a_vis_constraint);
opt_a_def = a_defs_iter.next(); opt_a_def = a_defs_iter.next();
opt_b_def = b_defs_iter.next(); opt_b_def = b_defs_iter.next();
} }
}, },
(Some(a_def), None) => { (Some(a_def), None) => {
// We've exhausted `b`, just push the def from `a` and move on to the next. // We've exhausted `b`, just push the def from `a` and move on to the next.
push(a_def, &mut a_constraints_iter, self); push(
a_def,
&mut a_constraints_iter,
&mut a_vis_constraints_iter,
self,
);
opt_a_def = a_defs_iter.next(); opt_a_def = a_defs_iter.next();
} }
(None, Some(b_def)) => { (None, Some(b_def)) => {
// We've exhausted `a`, just push the def from `b` and move on to the next. // We've exhausted `a`, just push the def from `b` and move on to the next.
push(b_def, &mut b_constraints_iter, self); push(
b_def,
&mut b_constraints_iter,
&mut b_vis_constraints_iter,
self,
);
opt_b_def = b_defs_iter.next(); opt_b_def = b_defs_iter.next();
} }
(None, None) => break, (None, None) => break,
} }
} }
// Same as above, but for declarations.
let mut a_decls_iter = a.declarations.live_declarations.iter();
let mut b_decls_iter = b.declarations.live_declarations.iter();
let mut a_vis_constraints_iter = a.declarations.visibility_constraints.into_iter();
let mut b_vis_constraints_iter = b.declarations.visibility_constraints.into_iter();
let mut opt_a_decl: Option<u32> = a_decls_iter.next();
let mut opt_b_decl: Option<u32> = b_decls_iter.next();
let push = |decl,
vis_constraints_iter: &mut VisibilityConstraintsIntoIterator,
merged: &mut Self| {
merged.declarations.live_declarations.insert(decl);
let vis_constraints = vis_constraints_iter
.next()
.expect("declarations and visibility_constraints length mismatch");
merged
.declarations
.visibility_constraints
.push(vis_constraints);
};
loop {
match (opt_a_decl, opt_b_decl) {
(Some(a_decl), Some(b_decl)) => match a_decl.cmp(&b_decl) {
std::cmp::Ordering::Less => {
push(a_decl, &mut a_vis_constraints_iter, self);
opt_a_decl = a_decls_iter.next();
}
std::cmp::Ordering::Greater => {
push(b_decl, &mut b_vis_constraints_iter, self);
opt_b_decl = b_decls_iter.next();
}
std::cmp::Ordering::Equal => {
push(a_decl, &mut b_vis_constraints_iter, self);
let a_vis_constraint = a_vis_constraints_iter
.next()
.expect("declarations and visibility_constraints length mismatch");
let current = self.declarations.visibility_constraints.last_mut().unwrap();
*current =
visibility_constraints.add_or_constraint(*current, a_vis_constraint);
opt_a_decl = a_decls_iter.next();
opt_b_decl = b_decls_iter.next();
}
},
(Some(a_decl), None) => {
push(a_decl, &mut a_vis_constraints_iter, self);
opt_a_decl = a_decls_iter.next();
}
(None, Some(b_decl)) => {
push(b_decl, &mut b_vis_constraints_iter, self);
opt_b_decl = b_decls_iter.next();
}
(None, None) => break,
}
}
} }
pub(super) fn bindings(&self) -> &SymbolBindings { pub(super) fn bindings(&self) -> &SymbolBindings {
@ -332,47 +507,44 @@ impl SymbolState {
pub(super) fn declarations(&self) -> &SymbolDeclarations { pub(super) fn declarations(&self) -> &SymbolDeclarations {
&self.declarations &self.declarations
} }
/// Could the symbol be unbound?
pub(super) fn may_be_unbound(&self) -> bool {
self.bindings.may_be_unbound()
}
}
/// The default state of a symbol, if we've seen no definitions of it, is undefined (that is,
/// both unbound and undeclared).
impl Default for SymbolState {
fn default() -> Self {
SymbolState::undefined()
}
} }
/// A single binding (as [`ScopedDefinitionId`]) with an iterator of its applicable /// A single binding (as [`ScopedDefinitionId`]) with an iterator of its applicable
/// [`ScopedConstraintId`]. /// narrowing constraints ([`ScopedConstraintId`]) and a corresponding visibility
/// visibility constraint ([`ScopedVisibilityConstraintId`]).
#[derive(Debug)] #[derive(Debug)]
pub(super) struct BindingIdWithConstraints<'a> { pub(super) struct BindingIdWithConstraints<'map> {
pub(super) definition: ScopedDefinitionId, pub(super) definition: ScopedDefinitionId,
pub(super) constraint_ids: ConstraintIdIterator<'a>, pub(super) constraint_ids: ConstraintIdIterator<'map>,
pub(super) visibility_constraint: ScopedVisibilityConstraintId,
} }
#[derive(Debug)] #[derive(Debug)]
pub(super) struct BindingIdWithConstraintsIterator<'a> { pub(super) struct BindingIdWithConstraintsIterator<'map> {
definitions: BindingsIterator<'a>, definitions: BindingsIterator<'map>,
constraints: ConstraintsIterator<'a>, constraints: ConstraintsIterator<'map>,
visibility_constraints: VisibilityConstraintsIterator<'map>,
} }
impl<'a> Iterator for BindingIdWithConstraintsIterator<'a> { impl<'map> Iterator for BindingIdWithConstraintsIterator<'map> {
type Item = BindingIdWithConstraints<'a>; type Item = BindingIdWithConstraints<'map>;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
match (self.definitions.next(), self.constraints.next()) { match (
(None, None) => None, self.definitions.next(),
(Some(def), Some(constraints)) => Some(BindingIdWithConstraints { self.constraints.next(),
definition: ScopedDefinitionId::from_u32(def), self.visibility_constraints.next(),
constraint_ids: ConstraintIdIterator { ) {
wrapped: constraints.iter(), (None, None, None) => None,
}, (Some(def), Some(constraints), Some(visibility_constraint_id)) => {
}), Some(BindingIdWithConstraints {
definition: ScopedDefinitionId::from_u32(def),
constraint_ids: ConstraintIdIterator {
wrapped: constraints.iter(),
},
visibility_constraint: *visibility_constraint_id,
})
}
// SAFETY: see above. // SAFETY: see above.
_ => unreachable!("definitions and constraints length mismatch"), _ => unreachable!("definitions and constraints length mismatch"),
} }
@ -396,16 +568,34 @@ impl Iterator for ConstraintIdIterator<'_> {
impl std::iter::FusedIterator for ConstraintIdIterator<'_> {} impl std::iter::FusedIterator for ConstraintIdIterator<'_> {}
/// A single declaration (as [`ScopedDefinitionId`]) with a corresponding visibility
/// visibility constraint ([`ScopedVisibilityConstraintId`]).
#[derive(Debug)] #[derive(Debug)]
pub(super) struct DeclarationIdIterator<'a> { pub(super) struct DeclarationIdWithConstraint {
inner: DeclarationsIterator<'a>, pub(super) definition: ScopedDefinitionId,
pub(super) visibility_constraint: ScopedVisibilityConstraintId,
}
pub(super) struct DeclarationIdIterator<'map> {
pub(crate) declarations: DeclarationsIterator<'map>,
pub(crate) visibility_constraints: VisibilityConstraintsIterator<'map>,
} }
impl Iterator for DeclarationIdIterator<'_> { impl Iterator for DeclarationIdIterator<'_> {
type Item = ScopedDefinitionId; type Item = DeclarationIdWithConstraint;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
self.inner.next().map(ScopedDefinitionId::from_u32) match (self.declarations.next(), self.visibility_constraints.next()) {
(None, None) => None,
(Some(declaration), Some(&visibility_constraint)) => {
Some(DeclarationIdWithConstraint {
definition: ScopedDefinitionId::from_u32(declaration),
visibility_constraint,
})
}
// SAFETY: see above.
_ => unreachable!("declarations and visibility_constraints length mismatch"),
}
} }
} }
@ -413,176 +603,172 @@ impl std::iter::FusedIterator for DeclarationIdIterator<'_> {}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{ScopedConstraintId, ScopedDefinitionId, SymbolState}; use super::*;
fn assert_bindings(symbol: &SymbolState, may_be_unbound: bool, expected: &[&str]) { #[track_caller]
assert_eq!(symbol.may_be_unbound(), may_be_unbound); fn assert_bindings(symbol: &SymbolState, expected: &[&str]) {
let actual = symbol let actual = symbol
.bindings() .bindings()
.iter() .iter()
.map(|def_id_with_constraints| { .map(|def_id_with_constraints| {
format!( let def_id = def_id_with_constraints.definition;
"{}<{}>", let def = if def_id == ScopedDefinitionId::UNBOUND {
def_id_with_constraints.definition.as_u32(), "unbound".into()
def_id_with_constraints } else {
.constraint_ids def_id.as_u32().to_string()
.map(ScopedConstraintId::as_u32) };
.map(|idx| idx.to_string()) let constraints = def_id_with_constraints
.collect::<Vec<_>>() .constraint_ids
.join(", ") .map(ScopedConstraintId::as_u32)
) .map(|idx| idx.to_string())
.collect::<Vec<_>>()
.join(", ");
format!("{def}<{constraints}>")
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
assert_eq!(actual, expected); assert_eq!(actual, expected);
} }
pub(crate) fn assert_declarations( #[track_caller]
symbol: &SymbolState, pub(crate) fn assert_declarations(symbol: &SymbolState, expected: &[&str]) {
may_be_undeclared: bool,
expected: &[u32],
) {
assert_eq!(symbol.declarations.may_be_undeclared(), may_be_undeclared);
let actual = symbol let actual = symbol
.declarations() .declarations()
.iter() .iter()
.map(ScopedDefinitionId::as_u32) .map(
|DeclarationIdWithConstraint {
definition,
visibility_constraint: _,
}| {
if definition == ScopedDefinitionId::UNBOUND {
"undeclared".into()
} else {
definition.as_u32().to_string()
}
},
)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
assert_eq!(actual, expected); assert_eq!(actual, expected);
} }
#[test] #[test]
fn unbound() { fn unbound() {
let sym = SymbolState::undefined(); let sym = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
assert_bindings(&sym, true, &[]); assert_bindings(&sym, &["unbound<>"]);
} }
#[test] #[test]
fn with() { fn with() {
let mut sym = SymbolState::undefined(); let mut sym = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym.record_binding(ScopedDefinitionId::from_u32(0)); sym.record_binding(ScopedDefinitionId::from_u32(1));
assert_bindings(&sym, false, &["0<>"]); assert_bindings(&sym, &["1<>"]);
}
#[test]
fn set_may_be_unbound() {
let mut sym = SymbolState::undefined();
sym.record_binding(ScopedDefinitionId::from_u32(0));
sym.set_may_be_unbound();
assert_bindings(&sym, true, &["0<>"]);
} }
#[test] #[test]
fn record_constraint() { fn record_constraint() {
let mut sym = SymbolState::undefined(); let mut sym = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym.record_binding(ScopedDefinitionId::from_u32(0)); sym.record_binding(ScopedDefinitionId::from_u32(1));
sym.record_constraint(ScopedConstraintId::from_u32(0)); sym.record_constraint(ScopedConstraintId::from_u32(0));
assert_bindings(&sym, false, &["0<0>"]); assert_bindings(&sym, &["1<0>"]);
} }
#[test] #[test]
fn merge() { fn merge() {
let mut visibility_constraints = VisibilityConstraints::default();
// merging the same definition with the same constraint keeps the constraint // merging the same definition with the same constraint keeps the constraint
let mut sym0a = SymbolState::undefined(); let mut sym1a = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym0a.record_binding(ScopedDefinitionId::from_u32(0)); sym1a.record_binding(ScopedDefinitionId::from_u32(1));
sym0a.record_constraint(ScopedConstraintId::from_u32(0)); sym1a.record_constraint(ScopedConstraintId::from_u32(0));
let mut sym0b = SymbolState::undefined(); let mut sym1b = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym0b.record_binding(ScopedDefinitionId::from_u32(0)); sym1b.record_binding(ScopedDefinitionId::from_u32(1));
sym0b.record_constraint(ScopedConstraintId::from_u32(0)); sym1b.record_constraint(ScopedConstraintId::from_u32(0));
sym0a.merge(sym0b); sym1a.merge(sym1b, &mut visibility_constraints);
let mut sym0 = sym0a; let mut sym1 = sym1a;
assert_bindings(&sym0, false, &["0<0>"]); assert_bindings(&sym1, &["1<0>"]);
// merging the same definition with differing constraints drops all constraints // merging the same definition with differing constraints drops all constraints
let mut sym1a = SymbolState::undefined(); let mut sym2a = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym1a.record_binding(ScopedDefinitionId::from_u32(1)); sym2a.record_binding(ScopedDefinitionId::from_u32(2));
sym1a.record_constraint(ScopedConstraintId::from_u32(1)); sym2a.record_constraint(ScopedConstraintId::from_u32(1));
let mut sym1b = SymbolState::undefined(); let mut sym1b = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym1b.record_binding(ScopedDefinitionId::from_u32(1)); sym1b.record_binding(ScopedDefinitionId::from_u32(2));
sym1b.record_constraint(ScopedConstraintId::from_u32(2)); sym1b.record_constraint(ScopedConstraintId::from_u32(2));
sym1a.merge(sym1b); sym2a.merge(sym1b, &mut visibility_constraints);
let sym1 = sym1a; let sym2 = sym2a;
assert_bindings(&sym1, false, &["1<>"]); assert_bindings(&sym2, &["2<>"]);
// merging a constrained definition with unbound keeps both // merging a constrained definition with unbound keeps both
let mut sym2a = SymbolState::undefined(); let mut sym3a = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym2a.record_binding(ScopedDefinitionId::from_u32(2)); sym3a.record_binding(ScopedDefinitionId::from_u32(3));
sym2a.record_constraint(ScopedConstraintId::from_u32(3)); sym3a.record_constraint(ScopedConstraintId::from_u32(3));
let sym2b = SymbolState::undefined(); let sym2b = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym2a.merge(sym2b); sym3a.merge(sym2b, &mut visibility_constraints);
let sym2 = sym2a; let sym3 = sym3a;
assert_bindings(&sym2, true, &["2<3>"]); assert_bindings(&sym3, &["unbound<>", "3<3>"]);
// merging different definitions keeps them each with their existing constraints // merging different definitions keeps them each with their existing constraints
sym0.merge(sym2); sym1.merge(sym3, &mut visibility_constraints);
let sym = sym0; let sym = sym1;
assert_bindings(&sym, true, &["0<0>", "2<3>"]); assert_bindings(&sym, &["unbound<>", "1<0>", "3<3>"]);
} }
#[test] #[test]
fn no_declaration() { fn no_declaration() {
let sym = SymbolState::undefined(); let sym = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
assert_declarations(&sym, true, &[]); assert_declarations(&sym, &["undeclared"]);
} }
#[test] #[test]
fn record_declaration() { fn record_declaration() {
let mut sym = SymbolState::undefined(); let mut sym = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym.record_declaration(ScopedDefinitionId::from_u32(1)); sym.record_declaration(ScopedDefinitionId::from_u32(1));
assert_declarations(&sym, false, &[1]); assert_declarations(&sym, &["1"]);
} }
#[test] #[test]
fn record_declaration_override() { fn record_declaration_override() {
let mut sym = SymbolState::undefined(); let mut sym = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym.record_declaration(ScopedDefinitionId::from_u32(1)); sym.record_declaration(ScopedDefinitionId::from_u32(1));
sym.record_declaration(ScopedDefinitionId::from_u32(2)); sym.record_declaration(ScopedDefinitionId::from_u32(2));
assert_declarations(&sym, false, &[2]); assert_declarations(&sym, &["2"]);
} }
#[test] #[test]
fn record_declaration_merge() { fn record_declaration_merge() {
let mut sym = SymbolState::undefined(); let mut visibility_constraints = VisibilityConstraints::default();
let mut sym = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym.record_declaration(ScopedDefinitionId::from_u32(1)); sym.record_declaration(ScopedDefinitionId::from_u32(1));
let mut sym2 = SymbolState::undefined(); let mut sym2 = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym2.record_declaration(ScopedDefinitionId::from_u32(2)); sym2.record_declaration(ScopedDefinitionId::from_u32(2));
sym.merge(sym2); sym.merge(sym2, &mut visibility_constraints);
assert_declarations(&sym, false, &[1, 2]); assert_declarations(&sym, &["1", "2"]);
} }
#[test] #[test]
fn record_declaration_merge_partial_undeclared() { fn record_declaration_merge_partial_undeclared() {
let mut sym = SymbolState::undefined(); let mut visibility_constraints = VisibilityConstraints::default();
let mut sym = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym.record_declaration(ScopedDefinitionId::from_u32(1)); sym.record_declaration(ScopedDefinitionId::from_u32(1));
let sym2 = SymbolState::undefined(); let sym2 = SymbolState::undefined(ScopedVisibilityConstraintId::ALWAYS_TRUE);
sym.merge(sym2); sym.merge(sym2, &mut visibility_constraints);
assert_declarations(&sym, true, &[1]); assert_declarations(&sym, &["undeclared", "1"]);
}
#[test]
fn set_may_be_undeclared() {
let mut sym = SymbolState::undefined();
sym.record_declaration(ScopedDefinitionId::from_u32(0));
sym.set_may_be_undeclared();
assert_declarations(&sym, true, &[0]);
} }
} }

2
crates/red_knot_python_semantic/src/symbol.rs
View File

@ -35,7 +35,7 @@ impl Boundness {
/// possibly_unbound: Symbol::Type(Type::IntLiteral(2), Boundness::PossiblyUnbound), /// possibly_unbound: Symbol::Type(Type::IntLiteral(2), Boundness::PossiblyUnbound),
/// non_existent: Symbol::Unbound, /// non_existent: Symbol::Unbound,
/// ``` /// ```
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Symbol<'db> { pub(crate) enum Symbol<'db> {
Type(Type<'db>, Boundness), Type(Type<'db>, Boundness),
Unbound, Unbound,

276
crates/red_knot_python_semantic/src/types.rs
View File

@ -23,7 +23,8 @@ use crate::semantic_index::definition::Definition;
use crate::semantic_index::symbol::{self as symbol, ScopeId, ScopedSymbolId}; use crate::semantic_index::symbol::{self as symbol, ScopeId, ScopedSymbolId};
use crate::semantic_index::{ use crate::semantic_index::{
global_scope, imported_modules, semantic_index, symbol_table, use_def_map, global_scope, imported_modules, semantic_index, symbol_table, use_def_map,
BindingWithConstraints, BindingWithConstraintsIterator, DeclarationsIterator, BindingWithConstraints, BindingWithConstraintsIterator, DeclarationWithConstraint,
DeclarationsIterator,
}; };
use crate::stdlib::{builtins_symbol, known_module_symbol, typing_extensions_symbol}; use crate::stdlib::{builtins_symbol, known_module_symbol, typing_extensions_symbol};
use crate::symbol::{Boundness, Symbol}; use crate::symbol::{Boundness, Symbol};
@ -68,6 +69,7 @@ pub fn check_types(db: &dyn Db, file: File) -> TypeCheckDiagnostics {
} }
/// Infer the public type of a symbol (its type as seen from outside its scope). /// Infer the public type of a symbol (its type as seen from outside its scope).
#[salsa::tracked]
fn symbol_by_id<'db>(db: &'db dyn Db, scope: ScopeId<'db>, symbol: ScopedSymbolId) -> Symbol<'db> { fn symbol_by_id<'db>(db: &'db dyn Db, scope: ScopeId<'db>, symbol: ScopedSymbolId) -> Symbol<'db> {
let _span = tracing::trace_span!("symbol_by_id", ?symbol).entered(); let _span = tracing::trace_span!("symbol_by_id", ?symbol).entered();
@ -75,51 +77,60 @@ fn symbol_by_id<'db>(db: &'db dyn Db, scope: ScopeId<'db>, symbol: ScopedSymbolI
// If the symbol is declared, the public type is based on declarations; otherwise, it's based // If the symbol is declared, the public type is based on declarations; otherwise, it's based
// on inference from bindings. // on inference from bindings.
if use_def.has_public_declarations(symbol) {
let declarations = use_def.public_declarations(symbol);
// If the symbol is undeclared in some paths, include the inferred type in the public type.
let undeclared_ty = if declarations.may_be_undeclared() {
Some(
bindings_ty(db, use_def.public_bindings(symbol))
.map(|bindings_ty| Symbol::Type(bindings_ty, use_def.public_boundness(symbol)))
.unwrap_or(Symbol::Unbound),
)
} else {
None
};
// Intentionally ignore conflicting declared types; that's not our problem, it's the
// problem of the module we are importing from.
// TODO: Our handling of boundness currently only depends on bindings, and ignores let declarations = use_def.public_declarations(symbol);
// declarations. This is inconsistent, since we only look at bindings if the symbol let declared = declarations_ty(db, declarations);
// may be undeclared. Consider the following example:
// ```py
// x: int
//
// if flag:
// y: int
// else
// y = 3
// ```
// If we import from this module, we will currently report `x` as a definitely-bound
// symbol (even though it has no bindings at all!) but report `y` as possibly-unbound
// (even though every path has either a binding or a declaration for it.)
match undeclared_ty { match declared {
Some(Symbol::Type(ty, boundness)) => Symbol::Type( // Symbol is declared, trust the declared type
declarations_ty(db, declarations, Some(ty)).unwrap_or_else(|(ty, _)| ty), Ok(symbol @ Symbol::Type(_, Boundness::Bound)) => symbol,
boundness, // Symbol is possibly declared
), Ok(Symbol::Type(declared_ty, Boundness::PossiblyUnbound)) => {
None | Some(Symbol::Unbound) => Symbol::Type( let bindings = use_def.public_bindings(symbol);
declarations_ty(db, declarations, None).unwrap_or_else(|(ty, _)| ty), let inferred = bindings_ty(db, bindings);
Boundness::Bound,
), match inferred {
// Symbol is possibly undeclared and definitely unbound
Symbol::Unbound => {
// TODO: We probably don't want to report `Bound` here. This requires a bit of
// design work though as we might want a different behavior for stubs and for
// normal modules.
Symbol::Type(declared_ty, Boundness::Bound)
}
// Symbol is possibly undeclared and (possibly) bound
Symbol::Type(inferred_ty, boundness) => Symbol::Type(
UnionType::from_elements(db, [inferred_ty, declared_ty].iter().copied()),
boundness,
),
}
}
// Symbol is undeclared, return the inferred type
Ok(Symbol::Unbound) => {
let bindings = use_def.public_bindings(symbol);
bindings_ty(db, bindings)
}
// Symbol is possibly undeclared
Err((declared_ty, _)) => {
// Intentionally ignore conflicting declared types; that's not our problem,
// it's the problem of the module we are importing from.
declared_ty.into()
} }
} else {
bindings_ty(db, use_def.public_bindings(symbol))
.map(|bindings_ty| Symbol::Type(bindings_ty, use_def.public_boundness(symbol)))
.unwrap_or(Symbol::Unbound)
} }
// TODO (ticket: https://github.com/astral-sh/ruff/issues/14297) Our handling of boundness
// currently only depends on bindings, and ignores declarations. This is inconsistent, since
// we only look at bindings if the symbol may be undeclared. Consider the following example:
// ```py
// x: int
//
// if flag:
// y: int
// else
// y = 3
// ```
// If we import from this module, we will currently report `x` as a definitely-bound symbol
// (even though it has no bindings at all!) but report `y` as possibly-unbound (even though
// every path has either a binding or a declaration for it.)
} }
/// Shorthand for `symbol_by_id` that takes a symbol name instead of an ID. /// Shorthand for `symbol_by_id` that takes a symbol name instead of an ID.
@ -132,6 +143,22 @@ fn symbol<'db>(db: &'db dyn Db, scope: ScopeId<'db>, name: &str) -> Symbol<'db>
{ {
return Symbol::Type(Type::BooleanLiteral(true), Boundness::Bound); return Symbol::Type(Type::BooleanLiteral(true), Boundness::Bound);
} }
if name == "platform"
&& file_to_module(db, scope.file(db))
.is_some_and(|module| module.is_known(KnownModule::Sys))
{
match Program::get(db).python_platform(db) {
crate::PythonPlatform::Identifier(platform) => {
return Symbol::Type(
Type::StringLiteral(StringLiteralType::new(db, platform.as_str())),
Boundness::Bound,
);
}
crate::PythonPlatform::All => {
// Fall through to the looked up type
}
}
}
let table = symbol_table(db, scope); let table = symbol_table(db, scope);
table table
@ -247,46 +274,77 @@ fn definition_expression_ty<'db>(
fn bindings_ty<'db>( fn bindings_ty<'db>(
db: &'db dyn Db, db: &'db dyn Db,
bindings_with_constraints: BindingWithConstraintsIterator<'_, 'db>, bindings_with_constraints: BindingWithConstraintsIterator<'_, 'db>,
) -> Option<Type<'db>> { ) -> Symbol<'db> {
let mut def_types = bindings_with_constraints.map( let visibility_constraints = bindings_with_constraints.visibility_constraints;
let mut bindings_with_constraints = bindings_with_constraints.peekable();
let unbound_visibility = if let Some(BindingWithConstraints {
binding: None,
constraints: _,
visibility_constraint,
}) = bindings_with_constraints.peek()
{
visibility_constraints.evaluate(db, *visibility_constraint)
} else {
Truthiness::AlwaysFalse
};
let mut types = bindings_with_constraints.filter_map(
|BindingWithConstraints { |BindingWithConstraints {
binding, binding,
constraints, constraints,
visibility_constraint,
}| { }| {
let binding = binding?;
let static_visibility = visibility_constraints.evaluate(db, visibility_constraint);
if static_visibility.is_always_false() {
return None;
}
let mut constraint_tys = constraints let mut constraint_tys = constraints
.filter_map(|constraint| narrowing_constraint(db, constraint, binding)) .filter_map(|constraint| narrowing_constraint(db, constraint, binding))
.peekable(); .peekable();
let binding_ty = binding_ty(db, binding); let binding_ty = binding_ty(db, binding);
if constraint_tys.peek().is_some() { if constraint_tys.peek().is_some() {
constraint_tys let intersection_ty = constraint_tys
.fold( .fold(
IntersectionBuilder::new(db).add_positive(binding_ty), IntersectionBuilder::new(db).add_positive(binding_ty),
IntersectionBuilder::add_positive, IntersectionBuilder::add_positive,
) )
.build() .build();
Some(intersection_ty)
} else { } else {
binding_ty Some(binding_ty)
} }
}, },
); );
if let Some(first) = def_types.next() { if let Some(first) = types.next() {
if let Some(second) = def_types.next() { let boundness = match unbound_visibility {
Some(UnionType::from_elements( Truthiness::AlwaysTrue => {
db, unreachable!("If we have at least one binding, the scope-start should not be definitely visible")
[first, second].into_iter().chain(def_types), }
)) Truthiness::AlwaysFalse => Boundness::Bound,
Truthiness::Ambiguous => Boundness::PossiblyUnbound,
};
if let Some(second) = types.next() {
Symbol::Type(
UnionType::from_elements(db, [first, second].into_iter().chain(types)),
boundness,
)
} else { } else {
Some(first) Symbol::Type(first, boundness)
} }
} else { } else {
None Symbol::Unbound
} }
} }
/// The result of looking up a declared type from declarations; see [`declarations_ty`]. /// The result of looking up a declared type from declarations; see [`declarations_ty`].
type DeclaredTypeResult<'db> = Result<Type<'db>, (Type<'db>, Box<[Type<'db>]>)>; type DeclaredTypeResult<'db> = Result<Symbol<'db>, (Type<'db>, Box<[Type<'db>]>)>;
/// Build a declared type from a [`DeclarationsIterator`]. /// Build a declared type from a [`DeclarationsIterator`].
/// ///
@ -304,40 +362,68 @@ type DeclaredTypeResult<'db> = Result<Type<'db>, (Type<'db>, Box<[Type<'db>]>)>;
fn declarations_ty<'db>( fn declarations_ty<'db>(
db: &'db dyn Db, db: &'db dyn Db,
declarations: DeclarationsIterator<'_, 'db>, declarations: DeclarationsIterator<'_, 'db>,
undeclared_ty: Option<Type<'db>>,
) -> DeclaredTypeResult<'db> { ) -> DeclaredTypeResult<'db> {
let mut declaration_types = declarations.map(|declaration| declaration_ty(db, declaration)); let visibility_constraints = declarations.visibility_constraints;
let mut declarations = declarations.peekable();
let Some(first) = declaration_types.next() else { let undeclared_visibility = if let Some(DeclarationWithConstraint {
if let Some(undeclared_ty) = undeclared_ty { declaration: None,
// Short-circuit to return the undeclared type if there are no declarations. visibility_constraint,
return Ok(undeclared_ty); }) = declarations.peek()
} {
panic!("declarations_ty must not be called with zero declarations and no undeclared_ty"); visibility_constraints.evaluate(db, *visibility_constraint)
} else {
Truthiness::AlwaysFalse
}; };
let mut conflicting: Vec<Type<'db>> = vec![]; let mut types = declarations.filter_map(
let mut builder = UnionBuilder::new(db).add(first); |DeclarationWithConstraint {
for other in declaration_types { declaration,
if !first.is_equivalent_to(db, other) { visibility_constraint,
conflicting.push(other); }| {
} let declaration = declaration?;
builder = builder.add(other); let static_visibility = visibility_constraints.evaluate(db, visibility_constraint);
}
// Avoid considering the undeclared type for the conflicting declaration diagnostics. It
// should still be part of the declared type.
if let Some(undeclared_ty) = undeclared_ty {
builder = builder.add(undeclared_ty);
}
let declared_ty = builder.build();
if conflicting.is_empty() { if static_visibility.is_always_false() {
Ok(declared_ty) None
} else {
Some(declaration_ty(db, declaration))
}
},
);
if let Some(first) = types.next() {
let mut conflicting: Vec<Type<'db>> = vec![];
let declared_ty = if let Some(second) = types.next() {
let mut builder = UnionBuilder::new(db).add(first);
for other in std::iter::once(second).chain(types) {
if !first.is_equivalent_to(db, other) {
conflicting.push(other);
}
builder = builder.add(other);
}
builder.build()
} else {
first
};
if conflicting.is_empty() {
let boundness = match undeclared_visibility {
Truthiness::AlwaysTrue => {
unreachable!("If we have at least one declaration, the scope-start should not be definitely visible")
}
Truthiness::AlwaysFalse => Boundness::Bound,
Truthiness::Ambiguous => Boundness::PossiblyUnbound,
};
Ok(Symbol::Type(declared_ty, boundness))
} else {
Err((
declared_ty,
std::iter::once(first).chain(conflicting).collect(),
))
}
} else { } else {
Err(( Ok(Symbol::Unbound)
declared_ty,
[first].into_iter().chain(conflicting).collect(),
))
} }
} }
@ -1587,7 +1673,7 @@ impl<'db> Type<'db> {
/// ///
/// This is used to determine the value that would be returned /// This is used to determine the value that would be returned
/// when `bool(x)` is called on an object `x`. /// when `bool(x)` is called on an object `x`.
fn bool(&self, db: &'db dyn Db) -> Truthiness { pub(crate) fn bool(&self, db: &'db dyn Db) -> Truthiness {
match self { match self {
Type::Any | Type::Todo(_) | Type::Never | Type::Unknown => Truthiness::Ambiguous, Type::Any | Type::Todo(_) | Type::Never | Type::Unknown => Truthiness::Ambiguous,
Type::FunctionLiteral(_) => Truthiness::AlwaysTrue, Type::FunctionLiteral(_) => Truthiness::AlwaysTrue,
@ -2866,11 +2952,19 @@ pub enum Truthiness {
} }
impl Truthiness { impl Truthiness {
const fn is_ambiguous(self) -> bool { pub(crate) const fn is_ambiguous(self) -> bool {
matches!(self, Truthiness::Ambiguous) matches!(self, Truthiness::Ambiguous)
} }
const fn negate(self) -> Self { pub(crate) const fn is_always_false(self) -> bool {
matches!(self, Truthiness::AlwaysFalse)
}
pub(crate) const fn is_always_true(self) -> bool {
matches!(self, Truthiness::AlwaysTrue)
}
pub(crate) const fn negate(self) -> Self {
match self { match self {
Self::AlwaysTrue => Self::AlwaysFalse, Self::AlwaysTrue => Self::AlwaysFalse,
Self::AlwaysFalse => Self::AlwaysTrue, Self::AlwaysFalse => Self::AlwaysTrue,
@ -2878,6 +2972,14 @@ impl Truthiness {
} }
} }
pub(crate) const fn negate_if(self, condition: bool) -> Self {
if condition {
self.negate()
} else {
self
}
}
fn into_type(self, db: &dyn Db) -> Type { fn into_type(self, db: &dyn Db) -> Type {
match self { match self {
Self::AlwaysTrue => Type::BooleanLiteral(true), Self::AlwaysTrue => Type::BooleanLiteral(true),

87
crates/red_knot_python_semantic/src/types/infer.rs
View File

@ -824,14 +824,10 @@ impl<'db> TypeInferenceBuilder<'db> {
debug_assert!(binding.is_binding(self.db())); debug_assert!(binding.is_binding(self.db()));
let use_def = self.index.use_def_map(binding.file_scope(self.db())); let use_def = self.index.use_def_map(binding.file_scope(self.db()));
let declarations = use_def.declarations_at_binding(binding); let declarations = use_def.declarations_at_binding(binding);
let undeclared_ty = if declarations.may_be_undeclared() {
Some(Type::Unknown)
} else {
None
};
let mut bound_ty = ty; let mut bound_ty = ty;
let declared_ty = declarations_ty(self.db(), declarations, undeclared_ty).unwrap_or_else( let declared_ty = declarations_ty(self.db(), declarations)
|(ty, conflicting)| { .map(|s| s.ignore_possibly_unbound().unwrap_or(Type::Unknown))
.unwrap_or_else(|(ty, conflicting)| {
// TODO point out the conflicting declarations in the diagnostic? // TODO point out the conflicting declarations in the diagnostic?
let symbol_table = self.index.symbol_table(binding.file_scope(self.db())); let symbol_table = self.index.symbol_table(binding.file_scope(self.db()));
let symbol_name = symbol_table.symbol(binding.symbol(self.db())).name(); let symbol_name = symbol_table.symbol(binding.symbol(self.db())).name();
@ -844,8 +840,7 @@ impl<'db> TypeInferenceBuilder<'db> {
), ),
); );
ty ty
}, });
);
if !bound_ty.is_assignable_to(self.db(), declared_ty) { if !bound_ty.is_assignable_to(self.db(), declared_ty) {
report_invalid_assignment(&self.context, node, declared_ty, bound_ty); report_invalid_assignment(&self.context, node, declared_ty, bound_ty);
// allow declarations to override inference in case of invalid assignment // allow declarations to override inference in case of invalid assignment
@ -860,7 +855,9 @@ impl<'db> TypeInferenceBuilder<'db> {
let use_def = self.index.use_def_map(declaration.file_scope(self.db())); let use_def = self.index.use_def_map(declaration.file_scope(self.db()));
let prior_bindings = use_def.bindings_at_declaration(declaration); let prior_bindings = use_def.bindings_at_declaration(declaration);
// unbound_ty is Never because for this check we don't care about unbound // unbound_ty is Never because for this check we don't care about unbound
let inferred_ty = bindings_ty(self.db(), prior_bindings).unwrap_or(Type::Never); let inferred_ty = bindings_ty(self.db(), prior_bindings)
.ignore_possibly_unbound()
.unwrap_or(Type::Never);
let ty = if inferred_ty.is_assignable_to(self.db(), ty) { let ty = if inferred_ty.is_assignable_to(self.db(), ty) {
ty ty
} else { } else {
@ -1739,7 +1736,9 @@ impl<'db> TypeInferenceBuilder<'db> {
guard, guard,
} = case; } = case;
self.infer_match_pattern(pattern); self.infer_match_pattern(pattern);
self.infer_optional_expression(guard.as_deref()); guard
.as_deref()
.map(|guard| self.infer_standalone_expression(guard));
self.infer_body(body); self.infer_body(body);
} }
} }
@ -1764,13 +1763,24 @@ impl<'db> TypeInferenceBuilder<'db> {
fn infer_match_pattern(&mut self, pattern: &ast::Pattern) { fn infer_match_pattern(&mut self, pattern: &ast::Pattern) {
// TODO(dhruvmanila): Add a Salsa query for inferring pattern types and matching against // TODO(dhruvmanila): Add a Salsa query for inferring pattern types and matching against
// the subject expression: https://github.com/astral-sh/ruff/pull/13147#discussion_r1739424510 // the subject expression: https://github.com/astral-sh/ruff/pull/13147#discussion_r1739424510
match pattern {
ast::Pattern::MatchValue(match_value) => {
self.infer_standalone_expression(&match_value.value);
}
_ => {
self.infer_match_pattern_impl(pattern);
}
}
}
fn infer_match_pattern_impl(&mut self, pattern: &ast::Pattern) {
match pattern { match pattern {
ast::Pattern::MatchValue(match_value) => { ast::Pattern::MatchValue(match_value) => {
self.infer_expression(&match_value.value); self.infer_expression(&match_value.value);
} }
ast::Pattern::MatchSequence(match_sequence) => { ast::Pattern::MatchSequence(match_sequence) => {
for pattern in &match_sequence.patterns { for pattern in &match_sequence.patterns {
self.infer_match_pattern(pattern); self.infer_match_pattern_impl(pattern);
} }
} }
ast::Pattern::MatchMapping(match_mapping) => { ast::Pattern::MatchMapping(match_mapping) => {
@ -1784,7 +1794,7 @@ impl<'db> TypeInferenceBuilder<'db> {
self.infer_expression(key); self.infer_expression(key);
} }
for pattern in patterns { for pattern in patterns {
self.infer_match_pattern(pattern); self.infer_match_pattern_impl(pattern);
} }
} }
ast::Pattern::MatchClass(match_class) => { ast::Pattern::MatchClass(match_class) => {
@ -1794,21 +1804,21 @@ impl<'db> TypeInferenceBuilder<'db> {
arguments, arguments,
} = match_class; } = match_class;
for pattern in &arguments.patterns { for pattern in &arguments.patterns {
self.infer_match_pattern(pattern); self.infer_match_pattern_impl(pattern);
} }
for keyword in &arguments.keywords { for keyword in &arguments.keywords {
self.infer_match_pattern(&keyword.pattern); self.infer_match_pattern_impl(&keyword.pattern);
} }
self.infer_expression(cls); self.infer_expression(cls);
} }
ast::Pattern::MatchAs(match_as) => { ast::Pattern::MatchAs(match_as) => {
if let Some(pattern) = &match_as.pattern { if let Some(pattern) = &match_as.pattern {
self.infer_match_pattern(pattern); self.infer_match_pattern_impl(pattern);
} }
} }
ast::Pattern::MatchOr(match_or) => { ast::Pattern::MatchOr(match_or) => {
for pattern in &match_or.patterns { for pattern in &match_or.patterns {
self.infer_match_pattern(pattern); self.infer_match_pattern_impl(pattern);
} }
} }
ast::Pattern::MatchStar(_) | ast::Pattern::MatchSingleton(_) => {} ast::Pattern::MatchStar(_) | ast::Pattern::MatchSingleton(_) => {}
@ -3094,28 +3104,24 @@ impl<'db> TypeInferenceBuilder<'db> {
let use_def = self.index.use_def_map(file_scope_id); let use_def = self.index.use_def_map(file_scope_id);
// If we're inferring types of deferred expressions, always treat them as public symbols // If we're inferring types of deferred expressions, always treat them as public symbols
let (bindings_ty, boundness) = if self.is_deferred() { let bindings_ty = if self.is_deferred() {
if let Some(symbol) = self.index.symbol_table(file_scope_id).symbol_id_by_name(id) { if let Some(symbol) = self.index.symbol_table(file_scope_id).symbol_id_by_name(id) {
( bindings_ty(self.db(), use_def.public_bindings(symbol))
bindings_ty(self.db(), use_def.public_bindings(symbol)),
use_def.public_boundness(symbol),
)
} else { } else {
assert!( assert!(
self.deferred_state.in_string_annotation(), self.deferred_state.in_string_annotation(),
"Expected the symbol table to create a symbol for every Name node" "Expected the symbol table to create a symbol for every Name node"
); );
(None, Boundness::PossiblyUnbound) Symbol::Unbound
} }
} else { } else {
let use_id = name.scoped_use_id(self.db(), self.scope()); let use_id = name.scoped_use_id(self.db(), self.scope());
( bindings_ty(self.db(), use_def.bindings_at_use(use_id))
bindings_ty(self.db(), use_def.bindings_at_use(use_id)),
use_def.use_boundness(use_id),
)
}; };
if boundness == Boundness::PossiblyUnbound { if let Symbol::Type(ty, Boundness::Bound) = bindings_ty {
ty
} else {
match self.lookup_name(name) { match self.lookup_name(name) {
Symbol::Type(looked_up_ty, looked_up_boundness) => { Symbol::Type(looked_up_ty, looked_up_boundness) => {
if looked_up_boundness == Boundness::PossiblyUnbound { if looked_up_boundness == Boundness::PossiblyUnbound {
@ -3123,20 +3129,22 @@ impl<'db> TypeInferenceBuilder<'db> {
} }
bindings_ty bindings_ty
.ignore_possibly_unbound()
.map(|ty| UnionType::from_elements(self.db(), [ty, looked_up_ty])) .map(|ty| UnionType::from_elements(self.db(), [ty, looked_up_ty]))
.unwrap_or(looked_up_ty) .unwrap_or(looked_up_ty)
} }
Symbol::Unbound => { Symbol::Unbound => match bindings_ty {
if bindings_ty.is_some() { Symbol::Type(ty, Boundness::PossiblyUnbound) => {
report_possibly_unresolved_reference(&self.context, name); report_possibly_unresolved_reference(&self.context, name);
} else { ty
report_unresolved_reference(&self.context, name);
} }
bindings_ty.unwrap_or(Type::Unknown) Symbol::Unbound => {
} report_unresolved_reference(&self.context, name);
Type::Unknown
}
Symbol::Type(_, Boundness::Bound) => unreachable!("Handled above"),
},
} }
} else {
bindings_ty.unwrap_or(Type::Unknown)
} }
} }
@ -6353,14 +6361,13 @@ mod tests {
} }
// Incremental inference tests // Incremental inference tests
#[track_caller]
fn first_public_binding<'db>(db: &'db TestDb, file: File, name: &str) -> Definition<'db> { fn first_public_binding<'db>(db: &'db TestDb, file: File, name: &str) -> Definition<'db> {
let scope = global_scope(db, file); let scope = global_scope(db, file);
use_def_map(db, scope) use_def_map(db, scope)
.public_bindings(symbol_table(db, scope).symbol_id_by_name(name).unwrap()) .public_bindings(symbol_table(db, scope).symbol_id_by_name(name).unwrap())
.next() .find_map(|b| b.binding)
.unwrap() .expect("no binding found")
.binding
} }
#[test] #[test]

41
crates/red_knot_python_semantic/src/types/narrow.rs
View File

@ -1,5 +1,7 @@
use crate::semantic_index::ast_ids::HasScopedExpressionId; use crate::semantic_index::ast_ids::HasScopedExpressionId;
use crate::semantic_index::constraint::{Constraint, ConstraintNode, PatternConstraint}; use crate::semantic_index::constraint::{
Constraint, ConstraintNode, PatternConstraint, PatternConstraintKind,
};
use crate::semantic_index::definition::Definition; use crate::semantic_index::definition::Definition;
use crate::semantic_index::expression::Expression; use crate::semantic_index::expression::Expression;
use crate::semantic_index::symbol::{ScopeId, ScopedSymbolId, SymbolTable}; use crate::semantic_index::symbol::{ScopeId, ScopedSymbolId, SymbolTable};
@ -216,31 +218,12 @@ impl<'db> NarrowingConstraintsBuilder<'db> {
) -> Option<NarrowingConstraints<'db>> { ) -> Option<NarrowingConstraints<'db>> {
let subject = pattern.subject(self.db); let subject = pattern.subject(self.db);
match pattern.pattern(self.db).node() { match pattern.kind(self.db) {
ast::Pattern::MatchValue(_) => { PatternConstraintKind::Singleton(singleton, _guard) => {
None // TODO self.evaluate_match_pattern_singleton(*subject, *singleton)
}
ast::Pattern::MatchSingleton(singleton_pattern) => {
self.evaluate_match_pattern_singleton(subject, singleton_pattern)
}
ast::Pattern::MatchSequence(_) => {
None // TODO
}
ast::Pattern::MatchMapping(_) => {
None // TODO
}
ast::Pattern::MatchClass(_) => {
None // TODO
}
ast::Pattern::MatchStar(_) => {
None // TODO
}
ast::Pattern::MatchAs(_) => {
None // TODO
}
ast::Pattern::MatchOr(_) => {
None // TODO
} }
// TODO: support more pattern kinds
PatternConstraintKind::Value(..) | PatternConstraintKind::Unsupported => None,
} }
} }
@ -483,14 +466,14 @@ impl<'db> NarrowingConstraintsBuilder<'db> {
fn evaluate_match_pattern_singleton( fn evaluate_match_pattern_singleton(
&mut self, &mut self,
subject: &ast::Expr, subject: Expression<'db>,
pattern: &ast::PatternMatchSingleton, singleton: ast::Singleton,
) -> Option<NarrowingConstraints<'db>> { ) -> Option<NarrowingConstraints<'db>> {
if let Some(ast::ExprName { id, .. }) = subject.as_name_expr() { if let Some(ast::ExprName { id, .. }) = subject.node_ref(self.db).as_name_expr() {
// SAFETY: we should always have a symbol for every Name node. // SAFETY: we should always have a symbol for every Name node.
let symbol = self.symbols().symbol_id_by_name(id).unwrap(); let symbol = self.symbols().symbol_id_by_name(id).unwrap();
let ty = match pattern.value { let ty = match singleton {
ast::Singleton::None => Type::none(self.db), ast::Singleton::None => Type::none(self.db),
ast::Singleton::True => Type::BooleanLiteral(true), ast::Singleton::True => Type::BooleanLiteral(true),
ast::Singleton::False => Type::BooleanLiteral(false), ast::Singleton::False => Type::BooleanLiteral(false),

338
crates/red_knot_python_semantic/src/visibility_constraints.rs Normal file
View File

@ -0,0 +1,338 @@
//! # Visibility constraints
//!
//! During semantic index building, we collect visibility constraints for each binding and
//! declaration. These constraints are then used during type-checking to determine the static
//! visibility of a certain definition. This allows us to re-analyze control flow during type
//! checking, potentially "hiding" some branches that we can statically determine to never be
//! taken. Consider the following example first. We added implicit "unbound" definitions at the
//! start of the scope. Note how visibility constraints can apply to bindings outside of the
//! if-statement:
//! ```py
//! x = <unbound> # not a live binding for the use of x below, shadowed by `x = 1`
//! y = <unbound> # visibility constraint: ~test
//!
//! x = 1 # visibility constraint: ~test
//! if test:
//! x = 2 # visibility constraint: test
//!
//! y = 2 # visibility constraint: test
//!
//! use(x)
//! use(y)
//! ```
//! The static truthiness of the `test` condition can either be always-false, ambiguous, or
//! always-true. Similarly, we have the same three options when evaluating a visibility constraint.
//! This outcome determines the visibility of a definition: always-true means that the definition
//! is definitely visible for a given use, always-false means that the definition is definitely
//! not visible, and ambiguous means that we might see this definition or not. In the latter case,
//! we need to consider both options during type inference and boundness analysis. For the example
//! above, these are the possible type inference / boundness results for the uses of `x` and `y`:
//!
//! ```text
//! | `test` truthiness | `~test` truthiness | type of `x` | boundness of `y` |
//! |-------------------|--------------------|-----------------|------------------|
//! | always false | always true | `Literal[1]` | unbound |
//! | ambiguous | ambiguous | `Literal[1, 2]` | possibly unbound |
//! | always true | always false | `Literal[2]` | bound |
//! ```
//!
//! ### Sequential constraints (ternary AND)
//!
//! As we have seen above, visibility constraints can apply outside of a control flow element.
//! So we need to consider the possibility that multiple constraints apply to the same binding.
//! Here, we consider what happens if multiple `if`-statements lead to a sequence of constraints.
//! Consider the following example:
//! ```py
//! x = 0
//!
//! if test1:
//! x = 1
//!
//! if test2:
//! x = 2
//! ```
//! The binding `x = 2` is easy to analyze. Its visibility corresponds to the truthiness of `test2`.
//! For the `x = 1` binding, things are a bit more interesting. It is always visible if `test1` is
//! always-true *and* `test2` is always-false. It is never visible if `test1` is always-false *or*
//! `test2` is always-true. And it is ambiguous otherwise. This corresponds to a ternary *test1 AND
//! ~test2* operation in three-valued Kleene logic [Kleene]:
//!
//! ```text
//! | AND | always-false | ambiguous | always-true |
//! |--------------|--------------|--------------|--------------|
//! | always false | always-false | always-false | always-false |
//! | ambiguous | always-false | ambiguous | ambiguous |
//! | always true | always-false | ambiguous | always-true |
//! ```
//!
//! The `x = 0` binding can be handled similarly, with the difference that both `test1` and `test2`
//! are negated:
//! ```py
//! x = 0 # ~test1 AND ~test2
//!
//! if test1:
//! x = 1 # test1 AND ~test2
//!
//! if test2:
//! x = 2 # test2
//! ```
//!
//! ### Merged constraints (ternary OR)
//!
//! Finally, we consider what happens in "parallel" control flow. Consider the following example
//! where we have omitted the test condition for the outer `if` for clarity:
//! ```py
//! x = 0
//!
//! if <…>:
//! if test1:
//! x = 1
//! else:
//! if test2:
//! x = 2
//!
//! use(x)
//! ```
//! At the usage of `x`, i.e. after control flow has been merged again, the visibility of the `x =
//! 0` binding behaves as follows: the binding is always visible if `test1` is always-false *or*
//! `test2` is always-false; and it is never visible if `test1` is always-true *and* `test2` is
//! always-true. This corresponds to a ternary *OR* operation in Kleene logic:
//!
//! ```text
//! | OR | always-false | ambiguous | always-true |
//! |--------------|--------------|--------------|--------------|
//! | always false | always-false | ambiguous | always-true |
//! | ambiguous | ambiguous | ambiguous | always-true |
//! | always true | always-true | always-true | always-true |
//! ```
//!
//! Using this, we can annotate the visibility constraints for the example above:
//! ```py
//! x = 0 # ~test1 OR ~test2
//!
//! if <…>:
//! if test1:
//! x = 1 # test1
//! else:
//! if test2:
//! x = 2 # test2
//!
//! use(x)
//! ```
//!
//! ### Explicit ambiguity
//!
//! In some cases, we explicitly add a `VisibilityConstraint::Ambiguous` constraint to all bindings
//! in a certain control flow path. We do this when branching on something that we can not (or
//! intentionally do not want to) analyze statically. `for` loops are one example:
//! ```py
//! x = <unbound>
//!
//! for _ in range(2):
//! x = 1
//! ```
//! Here, we report an ambiguous visibility constraint before branching off. If we don't do this,
//! the `x = <unbound>` binding would be considered unconditionally visible in the no-loop case.
//! And since the other branch does not have the live `x = <unbound>` binding, we would incorrectly
//! create a state where the `x = <unbound>` binding is always visible.
//!
//!
//! ### Properties
//!
//! The ternary `AND` and `OR` operations have the property that `~a OR ~b = ~(a AND b)`. This
//! means we could, in principle, get rid of either of these two to simplify the representation.
//!
//! However, we already apply negative constraints `~test1` and `~test2` to the "branches not
//! taken" in the example above. This means that the tree-representation `~test1 OR ~test2` is much
//! cheaper/shallower than basically creating `~(~(~test1) AND ~(~test2))`. Similarly, if we wanted
//! to get rid of `AND`, we would also have to create additional nodes. So for performance reasons,
//! there is a small "duplication" in the code between those two constraint types.
//!
//! [Kleene]: <https://en.wikipedia.org/wiki/Three-valued_logic#Kleene_and_Priest_logics>
use ruff_index::IndexVec;
use crate::semantic_index::ScopedVisibilityConstraintId;
use crate::semantic_index::{
ast_ids::HasScopedExpressionId,
constraint::{Constraint, ConstraintNode, PatternConstraintKind},
};
use crate::types::{infer_expression_types, Truthiness};
use crate::Db;
/// The maximum depth of recursion when evaluating visibility constraints.
///
/// This is a performance optimization that prevents us from descending deeply in case of
/// pathological cases. The actual limit here has been derived from performance testing on
/// the `black` codebase. When increasing the limit beyond 32, we see a 5x runtime increase
/// resulting from a few files with a lot of boolean expressions and `if`-statements.
const MAX_RECURSION_DEPTH: usize = 24;
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum VisibilityConstraint<'db> {
AlwaysTrue,
Ambiguous,
VisibleIf(Constraint<'db>),
VisibleIfNot(ScopedVisibilityConstraintId),
KleeneAnd(ScopedVisibilityConstraintId, ScopedVisibilityConstraintId),
KleeneOr(ScopedVisibilityConstraintId, ScopedVisibilityConstraintId),
}
#[derive(Debug, PartialEq, Eq)]
pub(crate) struct VisibilityConstraints<'db> {
constraints: IndexVec<ScopedVisibilityConstraintId, VisibilityConstraint<'db>>,
}
impl Default for VisibilityConstraints<'_> {
fn default() -> Self {
Self {
constraints: IndexVec::from_iter([VisibilityConstraint::AlwaysTrue]),
}
}
}
impl<'db> VisibilityConstraints<'db> {
pub(crate) fn add(
&mut self,
constraint: VisibilityConstraint<'db>,
) -> ScopedVisibilityConstraintId {
self.constraints.push(constraint)
}
pub(crate) fn add_or_constraint(
&mut self,
a: ScopedVisibilityConstraintId,
b: ScopedVisibilityConstraintId,
) -> ScopedVisibilityConstraintId {
match (&self.constraints[a], &self.constraints[b]) {
(_, VisibilityConstraint::VisibleIfNot(id)) if a == *id => {
ScopedVisibilityConstraintId::ALWAYS_TRUE
}
(VisibilityConstraint::VisibleIfNot(id), _) if *id == b => {
ScopedVisibilityConstraintId::ALWAYS_TRUE
}
_ => self.add(VisibilityConstraint::KleeneOr(a, b)),
}
}
pub(crate) fn add_and_constraint(
&mut self,
a: ScopedVisibilityConstraintId,
b: ScopedVisibilityConstraintId,
) -> ScopedVisibilityConstraintId {
if a == ScopedVisibilityConstraintId::ALWAYS_TRUE {
b
} else if b == ScopedVisibilityConstraintId::ALWAYS_TRUE {
a
} else {
self.add(VisibilityConstraint::KleeneAnd(a, b))
}
}
/// Analyze the statically known visibility for a given visibility constraint.
pub(crate) fn evaluate(&self, db: &'db dyn Db, id: ScopedVisibilityConstraintId) -> Truthiness {
self.evaluate_impl(db, id, MAX_RECURSION_DEPTH)
}
fn evaluate_impl(
&self,
db: &'db dyn Db,
id: ScopedVisibilityConstraintId,
max_depth: usize,
) -> Truthiness {
if max_depth == 0 {
return Truthiness::Ambiguous;
}
let visibility_constraint = &self.constraints[id];
match visibility_constraint {
VisibilityConstraint::AlwaysTrue => Truthiness::AlwaysTrue,
VisibilityConstraint::Ambiguous => Truthiness::Ambiguous,
VisibilityConstraint::VisibleIf(constraint) => Self::analyze_single(db, constraint),
VisibilityConstraint::VisibleIfNot(negated) => {
self.evaluate_impl(db, *negated, max_depth - 1).negate()
}
VisibilityConstraint::KleeneAnd(lhs, rhs) => {
let lhs = self.evaluate_impl(db, *lhs, max_depth - 1);
if lhs == Truthiness::AlwaysFalse {
return Truthiness::AlwaysFalse;
}
let rhs = self.evaluate_impl(db, *rhs, max_depth - 1);
if rhs == Truthiness::AlwaysFalse {
Truthiness::AlwaysFalse
} else if lhs == Truthiness::AlwaysTrue && rhs == Truthiness::AlwaysTrue {
Truthiness::AlwaysTrue
} else {
Truthiness::Ambiguous
}
}
VisibilityConstraint::KleeneOr(lhs_id, rhs_id) => {
let lhs = self.evaluate_impl(db, *lhs_id, max_depth - 1);
if lhs == Truthiness::AlwaysTrue {
return Truthiness::AlwaysTrue;
}
let rhs = self.evaluate_impl(db, *rhs_id, max_depth - 1);
if rhs == Truthiness::AlwaysTrue {
Truthiness::AlwaysTrue
} else if lhs == Truthiness::AlwaysFalse && rhs == Truthiness::AlwaysFalse {
Truthiness::AlwaysFalse
} else {
Truthiness::Ambiguous
}
}
}
}
fn analyze_single(db: &dyn Db, constraint: &Constraint) -> Truthiness {
match constraint.node {
ConstraintNode::Expression(test_expr) => {
let inference = infer_expression_types(db, test_expr);
let scope = test_expr.scope(db);
let ty =
inference.expression_ty(test_expr.node_ref(db).scoped_expression_id(db, scope));
ty.bool(db).negate_if(!constraint.is_positive)
}
ConstraintNode::Pattern(inner) => match inner.kind(db) {
PatternConstraintKind::Value(value, guard) => {
let subject_expression = inner.subject(db);
let inference = infer_expression_types(db, *subject_expression);
let scope = subject_expression.scope(db);
let subject_ty = inference.expression_ty(
subject_expression
.node_ref(db)
.scoped_expression_id(db, scope),
);
let inference = infer_expression_types(db, *value);
let scope = value.scope(db);
let value_ty =
inference.expression_ty(value.node_ref(db).scoped_expression_id(db, scope));
if subject_ty.is_single_valued(db) {
let truthiness =
Truthiness::from(subject_ty.is_equivalent_to(db, value_ty));
if truthiness.is_always_true() && guard.is_some() {
// Fall back to ambiguous, the guard might change the result.
Truthiness::Ambiguous
} else {
truthiness
}
} else {
Truthiness::Ambiguous
}
}
PatternConstraintKind::Singleton(..) | PatternConstraintKind::Unsupported => {
Truthiness::Ambiguous
}
},
}
}
}

13
crates/red_knot_test/src/config.rs
View File

@ -9,7 +9,7 @@
//! ``` //! ```
use anyhow::Context; use anyhow::Context;
use red_knot_python_semantic::PythonVersion; use red_knot_python_semantic::{PythonPlatform, PythonVersion};
use serde::Deserialize; use serde::Deserialize;
#[derive(Deserialize, Debug, Default, Clone)] #[derive(Deserialize, Debug, Default, Clone)]
@ -28,13 +28,22 @@ impl MarkdownTestConfig {
pub(crate) fn python_version(&self) -> Option<PythonVersion> { pub(crate) fn python_version(&self) -> Option<PythonVersion> {
self.environment.as_ref().and_then(|env| env.python_version) self.environment.as_ref().and_then(|env| env.python_version)
} }
pub(crate) fn python_platform(&self) -> Option<PythonPlatform> {
self.environment
.as_ref()
.and_then(|env| env.python_platform.clone())
}
} }
#[derive(Deserialize, Debug, Default, Clone)] #[derive(Deserialize, Debug, Default, Clone)]
#[serde(rename_all = "kebab-case", deny_unknown_fields)] #[serde(rename_all = "kebab-case", deny_unknown_fields)]
pub(crate) struct Environment { pub(crate) struct Environment {
/// Python version to assume when resolving types. /// Target Python version to assume when resolving types.
pub(crate) python_version: Option<PythonVersion>, pub(crate) python_version: Option<PythonVersion>,
/// Target platform to assume when resolving types.
pub(crate) python_platform: Option<PythonPlatform>,
} }
#[derive(Deserialize, Debug, Clone)] #[derive(Deserialize, Debug, Clone)]

5
crates/red_knot_test/src/db.rs
View File

@ -1,7 +1,7 @@
use red_knot_python_semantic::lint::RuleSelection; use red_knot_python_semantic::lint::RuleSelection;
use red_knot_python_semantic::{ use red_knot_python_semantic::{
default_lint_registry, Db as SemanticDb, Program, ProgramSettings, PythonVersion, default_lint_registry, Db as SemanticDb, Program, ProgramSettings, PythonPlatform,
SearchPathSettings, PythonVersion, SearchPathSettings,
}; };
use ruff_db::files::{File, Files}; use ruff_db::files::{File, Files};
use ruff_db::system::{DbWithTestSystem, System, SystemPath, SystemPathBuf, TestSystem}; use ruff_db::system::{DbWithTestSystem, System, SystemPath, SystemPathBuf, TestSystem};
@ -40,6 +40,7 @@ impl Db {
&db, &db,
&ProgramSettings { &ProgramSettings {
python_version: PythonVersion::default(), python_version: PythonVersion::default(),
python_platform: PythonPlatform::default(),
search_paths: SearchPathSettings::new(db.workspace_root.clone()), search_paths: SearchPathSettings::new(db.workspace_root.clone()),
}, },
) )

3
crates/red_knot_test/src/lib.rs
View File

@ -52,6 +52,9 @@ pub fn run(path: &Utf8Path, long_title: &str, short_title: &str, test_name: &str
Program::get(&db) Program::get(&db)
.set_python_version(&mut db) .set_python_version(&mut db)
.to(test.configuration().python_version().unwrap_or_default()); .to(test.configuration().python_version().unwrap_or_default());
Program::get(&db)
.set_python_platform(&mut db)
.to(test.configuration().python_platform().unwrap_or_default());
// Remove all files so that the db is in a "fresh" state. // Remove all files so that the db is in a "fresh" state.
db.memory_file_system().remove_all(); db.memory_file_system().remove_all();

5
crates/red_knot_workspace/src/workspace/settings.rs
View File

@ -1,5 +1,7 @@
use crate::workspace::PackageMetadata; use crate::workspace::PackageMetadata;
use red_knot_python_semantic::{ProgramSettings, PythonVersion, SearchPathSettings, SitePackages}; use red_knot_python_semantic::{
ProgramSettings, PythonPlatform, PythonVersion, SearchPathSettings, SitePackages,
};
use ruff_db::system::{SystemPath, SystemPathBuf}; use ruff_db::system::{SystemPath, SystemPathBuf};
/// The resolved configurations. /// The resolved configurations.
@ -40,6 +42,7 @@ impl Configuration {
WorkspaceSettings { WorkspaceSettings {
program: ProgramSettings { program: ProgramSettings {
python_version: self.python_version.unwrap_or_default(), python_version: self.python_version.unwrap_or_default(),
python_platform: PythonPlatform::default(),
search_paths: self.search_paths.to_settings(workspace_root), search_paths: self.search_paths.to_settings(workspace_root),
}, },
} }

2
crates/red_knot_workspace/src/workspace/snapshots/red_knot_workspace__workspace__metadata__tests__member_pattern_matching_file.snap
View File

@ -1,7 +1,6 @@
--- ---
source: crates/red_knot_workspace/src/workspace/metadata.rs source: crates/red_knot_workspace/src/workspace/metadata.rs
expression: "&workspace" expression: "&workspace"
snapshot_kind: text
--- ---
WorkspaceMetadata( WorkspaceMetadata(
root: "/app", root: "/app",
@ -23,6 +22,7 @@ WorkspaceMetadata(
settings: WorkspaceSettings( settings: WorkspaceSettings(
program: ProgramSettings( program: ProgramSettings(
python_version: "3.9", python_version: "3.9",
python_platform: all,
search_paths: SearchPathSettings( search_paths: SearchPathSettings(
extra_paths: [], extra_paths: [],
src_root: "/app", src_root: "/app",

2
crates/red_knot_workspace/src/workspace/snapshots/red_knot_workspace__workspace__metadata__tests__member_pattern_matching_hidden_folder.snap
View File

@ -1,7 +1,6 @@
--- ---
source: crates/red_knot_workspace/src/workspace/metadata.rs source: crates/red_knot_workspace/src/workspace/metadata.rs
expression: workspace expression: workspace
snapshot_kind: text
--- ---
WorkspaceMetadata( WorkspaceMetadata(
root: "/app", root: "/app",
@ -23,6 +22,7 @@ WorkspaceMetadata(
settings: WorkspaceSettings( settings: WorkspaceSettings(
program: ProgramSettings( program: ProgramSettings(
python_version: "3.9", python_version: "3.9",
python_platform: all,
search_paths: SearchPathSettings( search_paths: SearchPathSettings(
extra_paths: [], extra_paths: [],
src_root: "/app", src_root: "/app",

2
crates/red_knot_workspace/src/workspace/snapshots/red_knot_workspace__workspace__metadata__tests__package_without_pyproject.snap
View File

@ -1,7 +1,6 @@
--- ---
source: crates/red_knot_workspace/src/workspace/metadata.rs source: crates/red_knot_workspace/src/workspace/metadata.rs
expression: workspace expression: workspace
snapshot_kind: text
--- ---
WorkspaceMetadata( WorkspaceMetadata(
root: "/app", root: "/app",
@ -23,6 +22,7 @@ WorkspaceMetadata(
settings: WorkspaceSettings( settings: WorkspaceSettings(
program: ProgramSettings( program: ProgramSettings(
python_version: "3.9", python_version: "3.9",
python_platform: all,
search_paths: SearchPathSettings( search_paths: SearchPathSettings(
extra_paths: [], extra_paths: [],
src_root: "/app", src_root: "/app",

2
crates/red_knot_workspace/src/workspace/snapshots/red_knot_workspace__workspace__metadata__tests__single_package.snap
View File

@ -1,7 +1,6 @@
--- ---
source: crates/red_knot_workspace/src/workspace/metadata.rs source: crates/red_knot_workspace/src/workspace/metadata.rs
expression: workspace expression: workspace
snapshot_kind: text
--- ---
WorkspaceMetadata( WorkspaceMetadata(
root: "/app", root: "/app",
@ -23,6 +22,7 @@ WorkspaceMetadata(
settings: WorkspaceSettings( settings: WorkspaceSettings(
program: ProgramSettings( program: ProgramSettings(
python_version: "3.9", python_version: "3.9",
python_platform: all,
search_paths: SearchPathSettings( search_paths: SearchPathSettings(
extra_paths: [], extra_paths: [],
src_root: "/app", src_root: "/app",

2
crates/red_knot_workspace/src/workspace/snapshots/red_knot_workspace__workspace__metadata__tests__workspace_excluded.snap
View File

@ -1,7 +1,6 @@
--- ---
source: crates/red_knot_workspace/src/workspace/metadata.rs source: crates/red_knot_workspace/src/workspace/metadata.rs
expression: workspace expression: workspace
snapshot_kind: text
--- ---
WorkspaceMetadata( WorkspaceMetadata(
root: "/app", root: "/app",
@ -36,6 +35,7 @@ WorkspaceMetadata(
settings: WorkspaceSettings( settings: WorkspaceSettings(
program: ProgramSettings( program: ProgramSettings(
python_version: "3.9", python_version: "3.9",
python_platform: all,
search_paths: SearchPathSettings( search_paths: SearchPathSettings(
extra_paths: [], extra_paths: [],
src_root: "/app", src_root: "/app",

2
crates/red_knot_workspace/src/workspace/snapshots/red_knot_workspace__workspace__metadata__tests__workspace_members.snap
View File

@ -1,7 +1,6 @@
--- ---
source: crates/red_knot_workspace/src/workspace/metadata.rs source: crates/red_knot_workspace/src/workspace/metadata.rs
expression: workspace expression: workspace
snapshot_kind: text
--- ---
WorkspaceMetadata( WorkspaceMetadata(
root: "/app", root: "/app",
@ -49,6 +48,7 @@ WorkspaceMetadata(
settings: WorkspaceSettings( settings: WorkspaceSettings(
program: ProgramSettings( program: ProgramSettings(
python_version: "3.9", python_version: "3.9",
python_platform: all,
search_paths: SearchPathSettings( search_paths: SearchPathSettings(
extra_paths: [], extra_paths: [],
src_root: "/app", src_root: "/app",

5
crates/ruff_graph/src/db.rs
View File

@ -3,7 +3,9 @@ use std::sync::Arc;
use zip::CompressionMethod; use zip::CompressionMethod;
use red_knot_python_semantic::lint::RuleSelection; use red_knot_python_semantic::lint::RuleSelection;
use red_knot_python_semantic::{Db, Program, ProgramSettings, PythonVersion, SearchPathSettings}; use red_knot_python_semantic::{
Db, Program, ProgramSettings, PythonPlatform, PythonVersion, SearchPathSettings,
};
use ruff_db::files::{File, Files}; use ruff_db::files::{File, Files};
use ruff_db::system::{OsSystem, System, SystemPathBuf}; use ruff_db::system::{OsSystem, System, SystemPathBuf};
use ruff_db::vendored::{VendoredFileSystem, VendoredFileSystemBuilder}; use ruff_db::vendored::{VendoredFileSystem, VendoredFileSystemBuilder};
@ -49,6 +51,7 @@ impl ModuleDb {
&db, &db,
&ProgramSettings { &ProgramSettings {
python_version, python_version,
python_platform: PythonPlatform::default(),
search_paths, search_paths,
}, },
)?; )?;

3
fuzz/fuzz_targets/red_knot_check_invalid_syntax.rs
View File

@ -10,7 +10,7 @@ use libfuzzer_sys::{fuzz_target, Corpus};
use red_knot_python_semantic::types::check_types; use red_knot_python_semantic::types::check_types;
use red_knot_python_semantic::{ use red_knot_python_semantic::{
default_lint_registry, lint::RuleSelection, Db as SemanticDb, Program, ProgramSettings, default_lint_registry, lint::RuleSelection, Db as SemanticDb, Program, ProgramSettings,
PythonVersion, SearchPathSettings, PythonPlatform, PythonVersion, SearchPathSettings,
}; };
use ruff_db::files::{system_path_to_file, File, Files}; use ruff_db::files::{system_path_to_file, File, Files};
use ruff_db::system::{DbWithTestSystem, System, SystemPathBuf, TestSystem}; use ruff_db::system::{DbWithTestSystem, System, SystemPathBuf, TestSystem};
@ -112,6 +112,7 @@ fn setup_db() -> TestDb {
&db, &db,
&ProgramSettings { &ProgramSettings {
python_version: PythonVersion::default(), python_version: PythonVersion::default(),
python_platform: PythonPlatform::default(),
search_paths: SearchPathSettings::new(src_root), search_paths: SearchPathSettings::new(src_root),
}, },
) )