Python
/
pylyzer
mirror of https://github.com/mtshiba/pylyzer
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: support workspace diagnostics 

and fix some bugs
This commit is contained in:
Shunsuke Shibayama 2024-07-07 12:57:52 +09:00
parent 11b3940f32
commit 3c46a0340d
9 changed files with 116 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
Cargo.lock generated
View File

@ -119,7 +119,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustc_version",
"syn 2.0.68",
"syn 2.0.69",
]
[[package]]
@ -130,9 +130,9 @@ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
[[package]]
name = "els"
version = "0.1.52-nightly.0"
version = "0.1.52-nightly.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bd0a3c0bfdf681ff58dde3d15efc100f712df1ae1d7d9388cbd8107cb7e3c79"
checksum = "9f71553ed89956daa260ebefee8b4724308e8af507b713297aefa9535252048c"
dependencies = [
"erg_common",
"erg_compiler",
@ -144,9 +144,9 @@ dependencies = [
[[package]]
name = "erg_common"
version = "0.6.40-nightly.0"
version = "0.6.40-nightly.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c80f1574b5268d421f2067bdde5776f9363512706d011587432d2a0886eba56"
checksum = "891d4800e5dea9c2a52a56ad8af9654c292ad98afbcc3cec0480fea55cc468b0"
dependencies = [
"backtrace-on-stack-overflow",
"erg_proc_macros",
@ -156,9 +156,9 @@ dependencies = [
[[package]]
name = "erg_compiler"
version = "0.6.40-nightly.0"
version = "0.6.40-nightly.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efdec52061fad5d2238053cb15105d125e55cf116863e1b5337b8d286941ae3"
checksum = "a90c044a992d23a39eaf65b83485953935496aa1ae0d589b45c1611277d8cb54"
dependencies = [
"erg_common",
"erg_parser",
@ -166,9 +166,9 @@ dependencies = [
[[package]]
name = "erg_parser"
version = "0.6.40-nightly.0"
version = "0.6.40-nightly.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "926d6768a062c5851b20eda7f77aed41e7f326b8589ae4b8703b13c49364eab5"
checksum = "fb53c38fdc23f6e643267882c795040fda38da52309296106dba2e9dc544a31e"
dependencies = [
"erg_common",
"erg_proc_macros",
@ -177,9 +177,9 @@ dependencies = [
[[package]]
name = "erg_proc_macros"
version = "0.6.40-nightly.0"
version = "0.6.40-nightly.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da422e93bd4346cf04fadf410bc6b63881cda0842c1d976ee5a7f2b8ae29927e"
checksum = "93643cbe997e214daa35b54d4c948e4f4b1088866ba87004dc787f2e965f0f16"
dependencies = [
"quote",
"syn 1.0.109",
@ -248,7 +248,7 @@ dependencies = [
"Inflector",
"proc-macro2",
"quote",
"syn 2.0.68",
"syn 2.0.69",
]
[[package]]
@ -704,22 +704,22 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
[[package]]
name = "serde"
version = "1.0.203"
version = "1.0.204"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094"
checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.203"
version = "1.0.204"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba"
checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.68",
"syn 2.0.69",
]
[[package]]
@ -741,7 +741,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.68",
"syn 2.0.69",
]
[[package]]
@ -775,9 +775,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.68"
version = "2.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9"
checksum = "201fcda3845c23e8212cd466bfebf0bd20694490fc0356ae8e428e0824a915a6"
dependencies = [
"proc-macro2",
"quote",
@ -805,9 +805,9 @@ dependencies = [
[[package]]
name = "tinyvec"
version = "1.6.1"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82"
checksum = "ce6b6a2fb3a985e99cebfaefa9faa3024743da73304ca1c683a36429613d3d22"
dependencies = [
"tinyvec_macros",
]
@ -1030,5 +1030,5 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.68",
"syn 2.0.69",
]

6
Cargo.toml
View File

@ -22,9 +22,9 @@ edition = "2021"
repository = "https://github.com/mtshiba/pylyzer"
[workspace.dependencies]
erg_common = { version = "0.6.40-nightly.0", features = ["py_compat", "els"] }
erg_compiler = { version = "0.6.40-nightly.0", features = ["py_compat", "els"] }
els = { version = "0.1.52-nightly.0", features = ["py_compat"] }
erg_common = { version = "0.6.40-nightly.1", features = ["py_compat", "els"] }
erg_compiler = { version = "0.6.40-nightly.1", features = ["py_compat", "els"] }
els = { version = "0.1.52-nightly.1", features = ["py_compat"] }
# rustpython-parser = { version = "0.3.0", features = ["all-nodes-with-ranges", "location"] }
# rustpython-ast = { version = "0.3.0", features = ["all-nodes-with-ranges", "location"] }
rustpython-parser = { git = "https://github.com/RustPython/Parser", version = "0.3.1", features = ["all-nodes-with-ranges", "location"] }

42
crates/py2erg/convert.rs
View File

@ -2048,10 +2048,11 @@ impl ASTConverter {
DefBody::new(EQUAL, Block::new(vec![call]), DefId(0)),
)
} else {
self.register_name_info(&name.name, NameKind::Variable);
let top_module = name.name.split('.').next().unwrap();
self.register_name_info(top_module, NameKind::Variable);
let var = VarSignature::new(
VarPattern::Ident(
self.convert_ident(name.name.to_string(), name.location()),
self.convert_ident(top_module.to_string(), name.location()),
),
None,
);
@ -2159,20 +2160,29 @@ impl ASTConverter {
VarSignature::new(VarPattern::Ident(ident), None)
};
// from foo import bar, baz (if bar, baz is a module) ==> bar = import "foo/bar"; baz = import "foo/baz"
if let Ok(_path) = name_path {
let cont = format!("\"{module}/{}\"", name.name);
let mod_name = Expr::Literal(Literal::new(Token::new(
TokenKind::StrLit,
cont,
location.row.get(),
location.column.to_zero_indexed(),
)));
let call = import_acc.clone().call1(mod_name);
let def = Def::new(
Signature::Var(alias),
DefBody::new(EQUAL, Block::new(vec![call]), DefId(0)),
);
exprs.push(Expr::Def(def));
if let Ok(mut path) = name_path {
if path.ends_with("__init__.py") {
path.pop();
}
let mod_name = path.file_name().unwrap();
if name.name.as_str() == mod_name.to_string_lossy().trim_end_matches(".py") {
let cont = format!("\"{module}/{}\"", name.name);
let mod_name = Expr::Literal(Literal::new(Token::new(
TokenKind::StrLit,
cont,
location.row.get(),
location.column.to_zero_indexed(),
)));
let call = import_acc.clone().call1(mod_name);
let def = Def::new(
Signature::Var(alias),
DefBody::new(EQUAL, Block::new(vec![call]), DefId(0)),
);
exprs.push(Expr::Def(def));
} else {
// name.name: Foo, file_name: foo.py
imports.push(VarRecordAttr::new(true_name, alias));
}
} else {
imports.push(VarRecordAttr::new(true_name, alias));
}

73
crates/py2erg/gen_decl.rs
View File

@ -1,14 +1,14 @@
use std::fs::File;
use std::fs::{create_dir_all, File};
use std::io::{BufWriter, Write};
use std::path::Path;
use erg_common::io::Input;
use erg_common::pathutil::mod_name;
use erg_common::pathutil::{mod_name, NormalizedPathBuf};
use erg_common::set::Set;
use erg_common::traits::LimitedDisplay;
use erg_common::{log, Str};
use erg_compiler::build_package::{CheckStatus, PylyzerStatus};
use erg_compiler::hir::{ClassDef, Expr, HIR};
use erg_compiler::module::SharedModuleCache;
use erg_compiler::ty::value::{GenTypeObj, TypeObj};
use erg_compiler::ty::{HasType, Type};
@ -29,30 +29,33 @@ pub struct DeclFileGenerator {
}
impl DeclFileGenerator {
pub fn new(input: &Input, status: CheckStatus) -> Self {
pub fn new(path: &NormalizedPathBuf, status: CheckStatus) -> Self {
let (timestamp, hash) = {
let py_file_path = input.path();
let metadata = std::fs::metadata(py_file_path).unwrap();
let metadata = std::fs::metadata(path).unwrap();
let dummy_hash = metadata.len();
(metadata.modified().unwrap(), dummy_hash)
};
let status = PylyzerStatus {
status,
file: input.path().into(),
file: path.to_path_buf(),
timestamp,
hash,
};
let code = format!("{status}\n");
Self {
filename: input.filename().replace(".py", ".d.er"),
filename: path
.file_name()
.unwrap()
.to_string_lossy()
.replace(".py", ".d.er"),
namespace: "".to_string(),
imported: Set::new(),
code,
}
}
pub fn gen_decl_er(mut self, hir: HIR) -> DeclFile {
for chunk in hir.module.into_iter() {
pub fn gen_decl_er(mut self, hir: &HIR) -> DeclFile {
for chunk in hir.module.iter() {
self.gen_chunk_decl(chunk);
}
log!("code:\n{}", self.code);
@ -62,7 +65,7 @@ impl DeclFileGenerator {
}
}
fn gen_chunk_decl(&mut self, chunk: Expr) {
fn gen_chunk_decl(&mut self, chunk: &Expr) {
match chunk {
Expr::Def(def) => {
let mut name = def
@ -142,13 +145,13 @@ impl DeclFileGenerator {
self.code += &decl;
}
}
for attr in ClassDef::take_all_methods(def.methods_list) {
for attr in ClassDef::get_all_methods(&def.methods_list) {
self.gen_chunk_decl(attr);
}
self.namespace = stash;
}
Expr::Dummy(dummy) => {
for chunk in dummy.into_iter() {
for chunk in dummy.iter() {
self.gen_chunk_decl(chunk);
}
}
@ -158,31 +161,31 @@ impl DeclFileGenerator {
}
}
pub fn reserve_decl_er(input: Input) {
let mut dir = input.dir();
dir.push("__pycache__");
let pycache_dir = dir.as_path();
if !pycache_dir.exists() {
std::fs::create_dir(pycache_dir).unwrap();
fn dump_decl_er(path: &NormalizedPathBuf, hir: &HIR, status: CheckStatus) {
let decl_gen = DeclFileGenerator::new(path, status);
let file = decl_gen.gen_decl_er(hir);
let Some(dir) = path.parent().and_then(|p| p.canonicalize().ok()) else {
return;
};
let cache_dir = dir.join("__pycache__");
if !cache_dir.exists() {
let _ = create_dir_all(&cache_dir);
}
let filename = input.filename();
let mut path = pycache_dir.join(filename);
path.set_extension("d.er");
let path = cache_dir.join(file.filename);
if !path.exists() {
let _f = File::create(path).unwrap();
let _f = File::create(&path);
}
let Ok(f) = File::options().write(true).open(path) else {
return;
};
let mut f = BufWriter::new(f);
let _ = f.write_all(file.code.as_bytes());
}
pub fn dump_decl_er(input: Input, hir: HIR, status: CheckStatus) {
let decl_gen = DeclFileGenerator::new(&input, status);
let file = decl_gen.gen_decl_er(hir);
let mut dir = input.dir();
dir.push("__pycache__");
let pycache_dir = dir.as_path();
let f = File::options()
.write(true)
.open(pycache_dir.join(file.filename))
.unwrap();
let mut f = BufWriter::new(f);
f.write_all(file.code.as_bytes()).unwrap();
pub fn dump_decl_package(modules: &SharedModuleCache) {
for (path, module) in modules.raw_iter() {
if let Some(hir) = module.hir.as_ref() {
dump_decl_er(path, hir, module.status);
}
}
}

8
extension/package.json
View File

@ -3,7 +3,7 @@
"displayName": "pylyzer",
"description": "A fast Python static code analyzer & language server for VSCode",
"publisher": "pylyzer",
"version": "0.1.7",
"version": "0.1.8",
"engines": {
"vscode": "^1.70.0"
},
@ -14,7 +14,11 @@
},
"icon": "images/pylyzer-logo.png",
"main": "./dist/extension.js",
"activationEvents": ["onLanguage:python"],
"activationEvents": [
"workspaceContains:pyproject.toml",
"workspaceContains:*/pyproject.toml",
"onLanguage:python"
],
"contributes": {
"commands": [
{

28
src/analyze.rs
View File

@ -5,7 +5,7 @@ use erg_common::style::RESET;
use erg_common::traits::{ExitStatus, New, Runnable, Stream};
use erg_common::Str;
use erg_compiler::artifact::{BuildRunnable, Buildable, CompleteArtifact, IncompleteArtifact};
use erg_compiler::build_package::{CheckStatus, GenericPackageBuilder};
use erg_compiler::build_package::GenericPackageBuilder;
use erg_compiler::context::ModuleContext;
use erg_compiler::erg_parser::ast::{Module, AST};
use erg_compiler::erg_parser::build_ast::ASTBuildable;
@ -17,7 +17,7 @@ use erg_compiler::erg_parser::parse::Parsable;
use erg_compiler::error::{CompileError, CompileErrors};
use erg_compiler::module::SharedCompilerResource;
use erg_compiler::GenericHIRBuilder;
use py2erg::{dump_decl_er, reserve_decl_er, ShadowingMode};
use py2erg::{dump_decl_package, ShadowingMode};
use rustpython_ast::source_code::{RandomLocator, SourceRange};
use rustpython_ast::{Fold, ModModule};
use rustpython_parser::{Parse, ParseErrorType};
@ -267,13 +267,19 @@ impl PythonAnalyzer {
};
let erg_ast = AST::new(erg_common::Str::rc(&filename), erg_module);
erg_common::log!("AST:\n{erg_ast}");
self.check(erg_ast, errors, warns, mode)
let res = self.check(erg_ast, errors, warns, mode);
if self.cfg.mode.is_language_server() {
// mod_cache doesn't contains the current module
// we don't cache the current module's result for now
dump_decl_package(&self.checker.shared().mod_cache);
}
res
}
pub fn run(&mut self) {
if self.cfg.dist_dir.is_some() {
/*if self.cfg.dist_dir.is_some() {
reserve_decl_er(self.cfg.input.clone());
}
}*/
let py_code = self.cfg.input.read();
let filename = self.cfg.input.filename();
println!("{BLUE}Start checking{RESET}: {filename}");
@ -289,11 +295,7 @@ impl PythonAnalyzer {
}
println!("{GREEN}All checks OK{RESET}: {}", self.cfg.input.filename());
if self.cfg.dist_dir.is_some() {
dump_decl_er(
self.cfg.input.clone(),
artifact.object,
CheckStatus::Succeed,
);
dump_decl_package(&self.checker.shared().mod_cache);
println!("A declaration file has been generated to __pycache__ directory.");
}
std::process::exit(0);
@ -321,11 +323,7 @@ impl PythonAnalyzer {
};
// Even if type checking fails, some APIs are still valid, so generate a file
if self.cfg.dist_dir.is_some() {
dump_decl_er(
self.cfg.input.clone(),
artifact.object.unwrap(),
CheckStatus::Failed,
);
dump_decl_package(&self.checker.shared().mod_cache);
println!("A declaration file has been generated to __pycache__ directory.");
}
std::process::exit(code);

2
tests/foo/__init__.py
View File

@ -1,4 +1,4 @@
from .bar import i
from .bar import i, Bar
from . import bar
from . import baz

5
tests/foo/bar.py
View File

@ -1 +1,4 @@
i = 0
i = 0
class Bar:
def f(self): return 1

3
tests/import.py
View File

@ -1,7 +1,7 @@
import export
import foo
from . import foo
from foo import bar
from foo import bar, Bar
from foo import baz
import random
from random import randint as rdi
@ -28,6 +28,7 @@ assert d.x == 1
assert d.y == 2
assert foo.i == 0
assert Bar().f() == 1
from glob import glob
print(glob("*"))