From fddc571eea6d00f8e7af50e416f38f015fbf6ec2 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Sat, 5 Oct 2024 16:40:29 +0900 Subject: [PATCH] fix: typing/collections.abc types bug --- Cargo.lock | 20 ++++---- Cargo.toml | 6 +-- crates/py2erg/convert.rs | 100 +++++++++++++++++++++++++++++++++++---- tests/test.rs | 2 +- tests/typespec.py | 14 ++++++ 5 files changed, 118 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 89f9d2e..347b0e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -145,9 +145,9 @@ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "els" -version = "0.1.58-nightly.1" +version = "0.1.58-nightly.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed2c90d92d8be15be9e928f06d34e0cfe03c8c10e6351326859cecf3789dcac" +checksum = "5b8f4bd082ef9c4b0acd2557d89fdcf886a04355357255a1f4e8a04009ebc9de" dependencies = [ "erg_common", "erg_compiler", @@ -159,9 +159,9 @@ dependencies = [ [[package]] name = "erg_common" -version = "0.6.46-nightly.1" +version = "0.6.46-nightly.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4758c25017a49a7f3d8cb3287360deae39c696936a6747cf9e3d9f81cb94c010" +checksum = "cf40ea506598a316dfb4abe6ae9af54d6d3d2ebe8ab0a59c9e17506a96d4eb15" dependencies = [ "backtrace-on-stack-overflow", "erg_proc_macros", @@ -172,9 +172,9 @@ dependencies = [ [[package]] name = "erg_compiler" -version = "0.6.46-nightly.1" +version = "0.6.46-nightly.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c5c7fad1c74774dcbc293b79bb62a024135fcde4faf13411a3490761cb71a98" +checksum = "e58c92221e2dea780f3103d4ce14835d694aff8337ab0f8c184a25818a0f463f" dependencies = [ "erg_common", "erg_parser", @@ -182,9 +182,9 @@ dependencies = [ [[package]] name = "erg_parser" -version = "0.6.46-nightly.1" +version = "0.6.46-nightly.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c564e2914429af720277cb61256362762790da8c635558f77c4d6ae4c3a64f3a" +checksum = "f8df0a04d8e3ffd5c77d1d194ca37e1bd808a6a3b032b73bf861754544c30d57" dependencies = [ "erg_common", "erg_proc_macros", @@ -193,9 +193,9 @@ dependencies = [ [[package]] name = "erg_proc_macros" -version = "0.6.46-nightly.1" +version = "0.6.46-nightly.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fac38f9d18406130093186708186dad6f59efc04b0eddc0a8d0364be9361a90" +checksum = "bfaf0544746cc53a805a17dc61c4966802462d8151659109eb2050b023df004d" dependencies = [ "quote", "syn 1.0.109", diff --git a/Cargo.toml b/Cargo.toml index 9dcdd15..0036953 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,9 +24,9 @@ edition = "2021" repository = "https://github.com/mtshiba/pylyzer" [workspace.dependencies] -erg_common = { version = "0.6.46-nightly.1", features = ["py_compat", "els"] } -erg_compiler = { version = "0.6.46-nightly.1", features = ["py_compat", "els"] } -els = { version = "0.1.58-nightly.1", features = ["py_compat"] } +erg_common = { version = "0.6.46-nightly.2", features = ["py_compat", "els"] } +erg_compiler = { version = "0.6.46-nightly.2", features = ["py_compat", "els"] } +els = { version = "0.1.58-nightly.2", features = ["py_compat"] } # rustpython-parser = { version = "0.3.0", features = ["all-nodes-with-ranges", "location"] } # rustpython-ast = { version = "0.3.0", features = ["all-nodes-with-ranges", "location"] } rustpython-parser = { git = "https://github.com/RustPython/Parser", version = "0.4.0", features = ["all-nodes-with-ranges", "location"] } diff --git a/crates/py2erg/convert.rs b/crates/py2erg/convert.rs index e0c41be..683af1e 100644 --- a/crates/py2erg/convert.rs +++ b/crates/py2erg/convert.rs @@ -11,9 +11,9 @@ use erg_compiler::artifact::IncompleteArtifact; use erg_compiler::erg_parser::ast::{ Accessor, Args, BinOp, Block, ClassAttr, ClassAttrs, ClassDef, ConstAccessor, ConstApp, ConstArgs, ConstAttribute, ConstBinOp, ConstBlock, ConstDict, ConstExpr, ConstKeyValue, - ConstLambda, ConstList, ConstListWithLength, ConstNormalSet, ConstPosArg, ConstSet, Decorator, - Def, DefBody, DefId, DefaultParamSignature, Dict, Dummy, Expr, Identifier, KeyValue, KwArg, - Lambda, LambdaSignature, List, ListComprehension, Literal, Methods, Module, + ConstLambda, ConstList, ConstListWithLength, ConstNormalList, ConstNormalSet, ConstPosArg, + ConstSet, Decorator, Def, DefBody, DefId, DefaultParamSignature, Dict, Dummy, Expr, Identifier, + KeyValue, KwArg, Lambda, LambdaSignature, List, ListComprehension, Literal, Methods, Module, NonDefaultParamSignature, NormalDict, NormalList, NormalRecord, NormalSet, NormalTuple, ParamPattern, ParamTySpec, Params, PosArg, PreDeclTypeSpec, ReDef, Record, RecordAttrs, Set, SetComprehension, Signature, SubrSignature, SubrTypeSpec, Tuple, TupleTypeSpec, TypeAscription, @@ -39,6 +39,24 @@ use rustpython_parser::Parse; use crate::ast_util::accessor_name; use crate::error::*; +macro_rules! global_unary_collections { + () => { + "Collection" | "Container" | "Generator" | "Iterable" | "Iterator" | "Sequence" | "Set" + }; +} + +macro_rules! global_mutable_unary_collections { + () => { + "MutableSequence" | "MutableSet" | "MutableMapping" + }; +} + +macro_rules! global_binary_collections { + () => { + "Mapping" + }; +} + pub const ARROW: Token = Token::dummy(TokenKind::FuncArrow, "->"); #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -149,6 +167,19 @@ pub fn pyloc_to_ergloc(range: PySourceRange) -> erg_common::error::Location { ) } +pub fn ergloc_to_pyloc(loc: erg_common::error::Location) -> PySourceRange { + PySourceRange::new( + PyLocation { + row: OneIndexed::from_zero_indexed(loc.ln_begin().unwrap_or(0)), + column: OneIndexed::from_zero_indexed(loc.col_begin().unwrap_or(0)), + }, + PyLocation { + row: OneIndexed::from_zero_indexed(loc.ln_end().unwrap_or(0)), + column: OneIndexed::from_zero_indexed(loc.col_end().unwrap_or(0)), + }, + ) +} + fn attr_name_loc(value: &Expr) -> PyLocation { PyLocation { row: OneIndexed::from_zero_indexed(value.ln_end().unwrap_or(0)).saturating_sub(1), @@ -743,7 +774,43 @@ impl ASTConverter { } fn convert_ident_type_spec(&mut self, name: String, loc: PyLocation) -> TypeSpec { - TypeSpec::mono(self.convert_ident(name, loc)) + match &name[..] { + // Iterable[T] => Iterable(T), Iterable => Iterable(Obj) + global_unary_collections!() => TypeSpec::poly( + ConstExpr::Accessor(ConstAccessor::Local(Identifier::private("global".into()))) + .attr(Identifier::private(name.into())), + ConstArgs::single(ConstExpr::Accessor(ConstAccessor::Local( + Identifier::private("Obj".into()), + ))), + ), + // MutableSequence[T] => Sequence!(T), MutableSequence => Sequence!(Obj) + global_mutable_unary_collections!() => TypeSpec::poly( + ConstExpr::Accessor(ConstAccessor::Local(Identifier::private("global".into()))) + .attr(Identifier::private( + format!("{}!", name.trim_start_matches("Mutable")).into(), + )), + ConstArgs::single(ConstExpr::Accessor(ConstAccessor::Local( + Identifier::private("Obj".into()), + ))), + ), + // Mapping => Mapping(Obj, Obj) + global_binary_collections!() => TypeSpec::poly( + ConstExpr::Accessor(ConstAccessor::Local(Identifier::private("global".into()))) + .attr(Identifier::private(name.into())), + ConstArgs::pos_only( + vec![ + ConstPosArg::new(ConstExpr::Accessor(ConstAccessor::Local( + Identifier::private("Obj".into()), + ))), + ConstPosArg::new(ConstExpr::Accessor(ConstAccessor::Local( + Identifier::private("Obj".into()), + ))), + ], + None, + ), + ), + _ => TypeSpec::mono(self.convert_ident(name, loc)), + } } fn gen_dummy_type_spec(loc: PyLocation) -> TypeSpec { @@ -791,7 +858,7 @@ impl ASTConverter { ConstExpr::App(ConstApp::new(obj, app.attr_name, args)) } } - Some("GenericDict") => { + Some("GenericDict" | "Dict") => { if args.pos_args.len() == 2 { let key = args.pos_args.remove(0).expr; let value = args.pos_args.remove(0).expr; @@ -805,7 +872,7 @@ impl ASTConverter { ConstExpr::App(ConstApp::new(obj, app.attr_name, args)) } } - Some("GenericList") => { + Some("GenericList" | "List") => { if args.pos_args.len() == 2 { let elem = args.pos_args.remove(0).expr; let len = args.pos_args.remove(0).expr; @@ -821,7 +888,7 @@ impl ASTConverter { ConstExpr::App(ConstApp::new(obj, None, args)) } } - Some("GenericTuple") => { + Some("GenericTuple" | "Tuple") => { if args.pos_args.get(1).is_some_and(|arg| matches!(&arg.expr, ConstExpr::Lit(l) if l.is(TokenKind::EllipsisLit))) { let ty = args.pos_args.remove(0).expr; let obj = ConstExpr::Accessor(ConstAccessor::Local( @@ -833,6 +900,10 @@ impl ASTConverter { let obj = ConstExpr::Accessor(ConstAccessor::Local( Identifier::private("Tuple".into()), )); + let range = ergloc_to_pyloc(args.loc()); + let (l, r) = Self::gen_enclosure_tokens(TokenKind::LSqBr, range); + let list = ConstList::Normal(ConstNormalList::new(l, r, args, None)); + let args = ConstArgs::single(ConstExpr::List(list)); ConstExpr::App(ConstApp::new(obj, None, args)) } } @@ -1162,10 +1233,19 @@ impl ASTConverter { let namespace = Box::new(self.convert_expr(*attr.value)); let t = self.convert_ident(attr.attr.to_string(), attr_name_loc(&namespace)); if namespace - .get_name() - .is_some_and(|n| n == "typing" && t.inspect() == "Any") + .full_name() + .is_some_and(|n| n == "typing" || n == "collections.abc") { - return TypeSpec::PreDeclTy(PreDeclTypeSpec::Mono(t)); + match &t.inspect()[..] { + global_unary_collections!() + | global_mutable_unary_collections!() + | global_binary_collections!() => { + return self + .convert_ident_type_spec(attr.attr.to_string(), attr.range.start) + } + "Any" => return TypeSpec::PreDeclTy(PreDeclTypeSpec::Mono(t)), + _ => {} + } } let predecl = PreDeclTypeSpec::Attr { namespace, t }; TypeSpec::PreDeclTy(predecl) diff --git a/tests/test.rs b/tests/test.rs index df1d6fd..0657429 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -104,7 +104,7 @@ fn exec_warns() -> Result<(), String> { #[test] fn exec_typespec() -> Result<(), String> { - expect("tests/typespec.py", 0, 15) + expect("tests/typespec.py", 0, 16) } #[test] diff --git a/tests/typespec.py b/tests/typespec.py index a9a86e6..fe5151f 100644 --- a/tests/typespec.py +++ b/tests/typespec.py @@ -1,5 +1,6 @@ from typing import Union, Optional, Literal, Callable from collections.abc import Iterable, Mapping +import collections i: Union[int, str] = 1 # OK j: Union[int, str] = "aa" # OK @@ -10,6 +11,7 @@ p: Optional[int] = "a" # ERR weekdays: Literal[1, 2, 3, 4, 5, 6, 7] = 1 # OK weekdays: Literal[1, 2, 3, 4, 5, 6, 7] = 8 # ERR _: tuple[int, ...] = (1, 2, 3) +_: tuple[int, str] = (1, "a", 1) # OK, tuple[T, U, V] <: tuple[T, U] _: list[tuple[int, ...]] = [(1, 2, 3)] _: dict[str, dict[str, Union[int, str]]] = {"a": {"b": 1}} _: dict[str, dict[str, list[int]]] = {"a": {"b": [1]}} @@ -58,3 +60,15 @@ i1 = 1 # type: int i2 = 1 # type: str i3 = 1 # type: ignore i3 + "a" # OK + +def f(it: Iterable): + for i in it: + print(i) + +def f2(it: collections.abc.Iterable): + for i in it: + print(i) + +def g(it: Iterable): + for i in it: + print(i + "a") # ERR