Python
/
pylyzer
mirror of https://github.com/mtshiba/pylyzer
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

WIP: implement import resolving

This commit is contained in:
Shunsuke Shibayama 2022-12-14 08:09:12 +09:00
parent c17f6b7341
commit 85ca5765b6
12 changed files with 134 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1 +1,2 @@
/target
__pycache__/

8
Cargo.lock generated
View File

@ -206,7 +206,7 @@ checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
[[package]]
name = "els"
version = "0.1.11"
source = "git+https://github.com/erg-lang/erg-language-server?branch=main#be627c99faf539e8ad58f77eb41f8e76b759e6ab"
source = "git+https://github.com/erg-lang/erg-language-server?branch=pylyzer-mode#05c6849f2fca7c930e4366dc0a05281541d23da2"
dependencies = [
"erg_common",
"erg_compiler",
@ -227,7 +227,7 @@ dependencies = [
[[package]]
name = "erg_common"
version = "0.6.0-beta.2"
source = "git+https://github.com/erg-lang/erg?branch=main#2e289f7cd1503778293fa8c927df03825f6ef69b"
source = "git+https://github.com/erg-lang/erg?branch=pylyzer-mode#8528096a31c1e9c96eef93317b4f44890f779615"
dependencies = [
"hermit-abi",
"libc",
@ -237,7 +237,7 @@ dependencies = [
[[package]]
name = "erg_compiler"
version = "0.6.0-beta.2"
source = "git+https://github.com/erg-lang/erg?branch=main#2e289f7cd1503778293fa8c927df03825f6ef69b"
source = "git+https://github.com/erg-lang/erg?branch=pylyzer-mode#8528096a31c1e9c96eef93317b4f44890f779615"
dependencies = [
"erg_common",
"erg_parser",
@ -246,7 +246,7 @@ dependencies = [
[[package]]
name = "erg_parser"
version = "0.6.0-beta.2"
source = "git+https://github.com/erg-lang/erg?branch=main#2e289f7cd1503778293fa8c927df03825f6ef69b"
source = "git+https://github.com/erg-lang/erg?branch=pylyzer-mode#8528096a31c1e9c96eef93317b4f44890f779615"
dependencies = [
"erg_common",
"unicode-xid 0.2.4",

6
Cargo.toml
View File

@ -16,7 +16,7 @@ debug = ["erg_compiler/debug", "erg_common/debug"]
[dependencies]
rustpython-parser = "0.1.2"
erg_compiler = { git = "https://github.com/erg-lang/erg", branch = "main" }
erg_common = { git = "https://github.com/erg-lang/erg", branch = "main" }
els = { git = "https://github.com/erg-lang/erg-language-server", branch = "main" }
erg_compiler = { git = "https://github.com/erg-lang/erg", branch = "pylyzer-mode" }
erg_common = { git = "https://github.com/erg-lang/erg", branch = "pylyzer-mode" }
els = { git = "https://github.com/erg-lang/erg-language-server", branch = "pylyzer-mode" }
py2erg = { path = "./crates/py2erg" }

4
crates/py2erg/Cargo.toml
View File

@ -8,8 +8,8 @@ description = "A Python -> Erg converter"
[dependencies]
rustpython-parser = "0.1.2"
erg_compiler = { git = "https://github.com/erg-lang/erg", branch = "main" }
erg_common = { git = "https://github.com/erg-lang/erg", branch = "main" }
erg_compiler = { git = "https://github.com/erg-lang/erg", branch = "pylyzer-mode" }
erg_common = { git = "https://github.com/erg-lang/erg", branch = "pylyzer-mode" }
[lib]
path = "lib.rs"

75
crates/py2erg/convert.rs
View File

@ -7,36 +7,33 @@ use erg_compiler::erg_parser::token::{Token, TokenKind, EQUAL, COLON};
use erg_compiler::erg_parser::ast::{
Expr, Module, Signature, VarSignature, VarPattern, Params, Identifier, VarName, DefBody, DefId, Block, Def, Literal, Args, PosArg, Accessor,
BinOp, Lambda, LambdaSignature, TypeBoundSpecs, TypeSpec, SubrSignature, Decorator, NonDefaultParamSignature, DefaultParamSignature, ParamPattern, TypeSpecWithOp,
Tuple, NormalTuple, Array, NormalArray, Set, NormalSet, Dict, NormalDict, PreDeclTypeSpec, SimpleTypeSpec, ConstArgs, AttrDef, UnaryOp, KeyValue, Dummy
Tuple, NormalTuple, Array, NormalArray, Set, NormalSet, Dict, NormalDict, PreDeclTypeSpec, SimpleTypeSpec, ConstArgs, AttrDef, UnaryOp, KeyValue, Dummy, TypeAscription
};
fn add_procedural_mark(name: String) -> String {
/// Variables are automatically rewritten with `python_compatible_mode`,
/// but types are rewritten here because they are complex components used inseparably in the Erg system.
fn escape_name(name: String) -> String {
match &name[..] {
"print" => "print!".to_string(),
"input" => "input!".to_string(),
"open" => "open!".to_string(),
"int" => "Int".to_string(),
"float" => "Float".to_string(),
"str" => "Str".to_string(),
"bool" => "Bool".to_string(),
"list" => "GenericArray".to_string(),
"tuple" => "GenericTuple".to_string(),
"range" => "Range".to_string(),
"int" => "Int".into(),
"float" => "Float".into(),
"str" => "Str".into(),
"bool" => "Bool".into(),
"list" => "GenericArray".into(),
"range" => "GenericRange".into(),
"dict" => "GenericDict".into(),
"set" => "GenericSet".into(),
"tuple" => "GenericTuple".into(),
"type" => "Type".into(),
"ModuleType" => "GeneticModule".into(),
_ => name,
}
}
fn convert_attr(name: String, loc: PyLocation) -> Identifier {
let token = Token::new(TokenKind::Symbol, add_procedural_mark(name), loc.row(), loc.column() - 1);
let name = VarName::new(token);
let dot = Token::new(TokenKind::Dot, ".", loc.row(), loc.column());
Identifier::new(Some(dot), name)
}
fn convert_ident(name: String, loc: PyLocation) -> Identifier {
let token = Token::new(TokenKind::Symbol, add_procedural_mark(name), loc.row(), loc.column() - 1);
let token = Token::new(TokenKind::Symbol, escape_name(name), loc.row(), loc.column() - 1);
let name = VarName::new(token);
Identifier::new(None, name)
let dot = Token::new(TokenKind::Dot, ".", loc.row(), loc.column() - 1);
Identifier::new(Some(dot), name)
}
fn convert_param_pattern(arg: String, loc: PyLocation) -> ParamPattern {
@ -183,7 +180,7 @@ fn convert_expr(expr: Located<ExpressionType>) -> Expr {
}
ExpressionType::Attribute { value, name } => {
let obj = convert_expr(*value);
let name = convert_attr(name, expr.location);
let name = convert_ident(name, expr.location);
Expr::Accessor(Accessor::attr(obj, name))
}
ExpressionType::Lambda { args, body } => {
@ -226,7 +223,7 @@ fn convert_expr(expr: Located<ExpressionType>) -> Expr {
}
ExpressionType::Subscript { a, b } => {
let obj = convert_expr(*a);
let method = obj.attr_expr(convert_attr("__getitem__".to_string(), expr.location));
let method = obj.attr_expr(convert_ident("__getitem__".to_string(), expr.location));
let args = Args::new(vec![PosArg::new(convert_expr(*b))], vec![], None);
method.call_expr(args)
}
@ -244,6 +241,36 @@ fn convert_block(block: Vec<Located<StatementType>>) -> Block {
fn convert_statement(stmt: Located<StatementType>) -> Expr {
match stmt.node {
StatementType::Expression { expression } => convert_expr(expression),
StatementType::AnnAssign { target, annotation, value } => {
let t_spec = convert_type_spec(*annotation);
match target.node {
ExpressionType::Identifier { name } => {
let ident = convert_ident(name, stmt.location);
if let Some(value) = value {
let sig = Signature::Var(VarSignature::new(VarPattern::Ident(ident), Some(t_spec)));
let block = Block::new(vec![convert_expr(value)]);
let body = DefBody::new(EQUAL, block, DefId(0));
let def = Def::new(sig, body);
Expr::Def(def)
} else {
let tasc = TypeAscription::new(Expr::Accessor(Accessor::Ident(ident)), COLON, t_spec);
Expr::TypeAsc(tasc)
}
}
ExpressionType::Attribute { value: attr, name } => {
let attr = convert_expr(*attr).attr(convert_ident(name, target.location));
if let Some(value) = value {
let expr = convert_expr(value);
let adef = AttrDef::new(attr, expr);
Expr::AttrDef(adef)
} else {
let tasc = TypeAscription::new(Expr::Accessor(attr), COLON, t_spec);
Expr::TypeAsc(tasc)
}
}
_other => Expr::Dummy(Dummy::empty()),
}
}
StatementType::Assign { mut targets, value } => {
let lhs = targets.remove(0);
match lhs.node {
@ -339,7 +366,7 @@ fn convert_statement(stmt: Located<StatementType>) -> Expr {
StatementType::Import { names } => {
let mut imports = vec![];
for name in names {
let import_acc = Expr::Accessor(Accessor::Ident(convert_ident("pyimport".to_string(), stmt.location)));
let import_acc = Expr::Accessor(Accessor::Ident(convert_ident("__import__".to_string(), stmt.location)));
let cont = format!("\"{}\"", name.symbol);
let mod_name = Expr::Lit(Literal::new(Token::new(TokenKind::StrLit, cont, stmt.location.row(), stmt.location.column() - 1)));
let args = Args::new(vec![PosArg::new(mod_name)], vec![], None);

45
crates/py2erg/gen_decl.rs Normal file
View File

@ -0,0 +1,45 @@
use std::io::Write;
use erg_compiler::hir::{HIR, Expr};
use erg_compiler::ty::HasType;
pub struct DeclFile {
pub filename: String,
pub code: String,
}
fn escape_type(typ: String) -> String {
typ.replace('%', "Type_")
}
pub fn gen_decl_er(hir: HIR) -> DeclFile {
let mut code = "".to_string();
for chunk in hir.module.into_iter() {
match chunk {
Expr::Def(def) => {
let typ = def.sig.ident().ref_t().to_string();
let typ = escape_type(typ);
let decl = format!(".{}: {typ}", def.sig.ident().inspect());
code += &decl;
}
Expr::ClassDef(def) => {
let decl = format!(".{}: ClassType", def.sig.ident().inspect());
code += &decl;
}
_ => {}
}
code.push('\n');
}
let filename = hir.name.replace(".py", ".d.er");
DeclFile { filename, code }
}
pub fn dump_decl_er(hir: HIR) {
let file = gen_decl_er(hir);
if !std::path::Path::new("__pycache__").exists() {
std::fs::create_dir("__pycache__").unwrap();
}
let f = std::fs::File::create(format!("__pycache__/{}", file.filename)).unwrap();
let mut f = std::io::BufWriter::new(f);
f.write_all(file.code.as_bytes()).unwrap();
}

3
crates/py2erg/lib.rs
View File

@ -1,2 +1,5 @@
mod convert;
mod gen_decl;
pub use convert::*;
pub use gen_decl::*;

5
src/analyze.rs
View File

@ -6,6 +6,7 @@ use erg_compiler::context::Context;
use erg_compiler::erg_parser::ast::AST;
use erg_compiler::error::{CompileErrors, CompileError};
use erg_compiler::lower::ASTLowerer;
use py2erg::dump_decl_er;
use rustpython_parser::parser;
use crate::handle_err;
@ -85,6 +86,10 @@ impl PythonAnalyzer {
artifact.warns.fmt_all_stderr();
}
println!("All checks OK.");
if self.cfg.output_dir.is_some() {
dump_decl_er(artifact.object);
println!("A declaration file has been generated to __pycache__ directory.");
}
std::process::exit(0);
}
Err(artifact) => {

20
src/handle_err.rs
View File

@ -1,5 +1,5 @@
use erg_common::error::ErrorKind;
use erg_common::style::{remove_style, StyledString, Color};
// use erg_common::style::{remove_style, StyledString, Color};
use erg_compiler::error::{CompileErrors, CompileError};
use erg_compiler::context::Context;
@ -7,30 +7,14 @@ pub(crate) fn filter_errors(ctx: &Context, errors: CompileErrors) -> CompileErro
errors.into_iter().filter_map(|error| filter_error(ctx, error)).collect()
}
fn filter_error(ctx: &Context, error: CompileError) -> Option<CompileError> {
fn filter_error(_ctx: &Context, error: CompileError) -> Option<CompileError> {
match error.core.kind {
ErrorKind::VisibilityError => None,
ErrorKind::NameError => Some(map_name_error(ctx, error)),
ErrorKind::AssignError => handle_assign_error(error),
_ => Some(error),
}
}
fn map_name_error(ctx: &Context, mut error: CompileError) -> CompileError {
let hint = error.core.sub_messages.iter_mut().find_map(|sub| sub.hint.as_mut());
if let Some(hint) = hint {
if let Some(name) = hint.split("exists a similar name variable: ").last() {
let name = remove_style(name);
if let Some((_, vi)) = ctx.get_var_info(&name) {
if let Some(py_name) = &vi.py_name {
*hint = format!("exists a similar name variable: {}", StyledString::new(&py_name[..], Some(Color::Green), None));
}
}
}
}
error
}
fn handle_assign_error(error: CompileError) -> Option<CompileError> {
if error.core.main_message.ends_with("cannot be assigned more than once") {
None

12
src/main.rs
View File

@ -8,12 +8,13 @@ use std::str::FromStr;
use analyze::PythonAnalyzer;
use els::Server;
use erg_common::config::{Input, ErgConfig};
use erg_common::spawn::exec_new_thread;
use erg_common::traits::Runnable;
pub fn parse_args() -> ErgConfig {
let mut args = env::args();
args.next(); // "pylyzer"
let mut cfg = ErgConfig::default();
let mut cfg = ErgConfig{ python_compatible_mode: true, ..ErgConfig::default() };
while let Some(arg) = args.next() {
match &arg[..] {
"--" => {
@ -28,6 +29,9 @@ pub fn parse_args() -> ErgConfig {
"--server" => {
cfg.mode = "server";
}
"--dump-decl" => {
cfg.output_dir = Some("");
}
"--verbose" => {
cfg.verbose = args
.next()
@ -68,7 +72,7 @@ For more information try `pylyzer --help`"
cfg
}
fn main() {
fn run() {
let cfg = parse_args();
if cfg.mode == "server" {
let mut lang_server = Server::<PythonAnalyzer>::new();
@ -80,3 +84,7 @@ fn main() {
analyzer.run();
}
}
fn main() {
exec_new_thread(run);
}

1
tests/export.py Normal file
View File

@ -0,0 +1 @@
x = 1

8
tests/import.py
View File

@ -1,4 +1,10 @@
import export
import random
i = random.randint(0, 1)
print(i + "aa")
print(i + 1)
print(export.x)
def add(a, b):
return a + b