From 69d634115aa492b2ac2ddf623ad9f883e3c9d437 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 29 May 2023 00:55:17 +0900 Subject: [PATCH 1/8] trial --- Cargo.lock | 704 +++++++++++--------------------------- Cargo.toml | 4 +- crates/py2erg/Cargo.toml | 1 + crates/py2erg/ast_util.rs | 122 +------ crates/py2erg/clone.rs | 272 --------------- crates/py2erg/convert.rs | 344 ++++++++++--------- crates/py2erg/lib.rs | 1 - src/analyze.rs | 9 +- 8 files changed, 383 insertions(+), 1074 deletions(-) delete mode 100644 crates/py2erg/clone.rs diff --git a/Cargo.lock b/Cargo.lock index 07819c3..064389d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" + [[package]] name = "addr2line" version = "0.19.0" @@ -18,54 +24,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] -name = "aho-corasick" -version = "1.0.1" +name = "anyhow" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" -dependencies = [ - "memchr", -] - -[[package]] -name = "arrayref" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" - -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - -[[package]] -name = "ascii-canvas" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8eb72df928aafb99fe5d37b383f2fe25bd2a765e3e5f7c365916b6f2463a29" -dependencies = [ - "term", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - -[[package]] -name = "autocfg" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" -dependencies = [ - "autocfg 1.1.0", -] +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" [[package]] name = "autocfg" @@ -99,77 +61,12 @@ dependencies = [ "nix", ] -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "bit-set" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "blake2b_simd" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" -dependencies = [ - "arrayref", - "arrayvec", - "constant_time_eq", -] - -[[package]] -name = "block-buffer" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" -dependencies = [ - "block-padding", - "byte-tools", - "byteorder", - "generic-array", -] - -[[package]] -name = "block-padding" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -dependencies = [ - "byte-tools", -] - -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" - -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - [[package]] name = "cc" version = "1.0.79" @@ -183,66 +80,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "cloudabi" -version = "0.0.3" +name = "crunchy" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags", -] - -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - -[[package]] -name = "crossbeam-utils" -version = "0.8.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - -[[package]] -name = "digest" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" -dependencies = [ - "generic-array", -] - -[[package]] -name = "dirs" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - -[[package]] -name = "docopt" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f3f119846c823f9eafcf953a8f6ffb6ed69bf6240883261a7f13b634579a51f" -dependencies = [ - "lazy_static", - "regex", - "serde", - "strsim", -] +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "either" @@ -263,15 +104,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "ena" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8944dc8fa28ce4a38f778bd46bf7d923fe73eed5a439398507246c8e017e6f36" -dependencies = [ - "log", -] - [[package]] name = "erg_common" version = "0.6.14-nightly.0" @@ -301,21 +133,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfd44c03719e8051c7997263298e2b792da6138f818646d0eeae359f92bf0c60" dependencies = [ "erg_common", - "unicode-xid 0.2.4", + "unicode-xid", ] -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" - -[[package]] -name = "fixedbitset" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" - [[package]] name = "form_urlencoded" version = "1.1.0" @@ -325,32 +145,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - -[[package]] -name = "generic-array" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" -dependencies = [ - "typenum", -] - -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - [[package]] name = "gimli" version = "0.27.2" @@ -366,6 +160,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hexf-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + [[package]] name = "idna" version = "0.3.0" @@ -377,10 +177,23 @@ dependencies = [ ] [[package]] -name = "itertools" -version = "0.8.2" +name = "is-macro" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" +checksum = "8a7d079e129b77477a49c5c4f1cfe9ce6c2c909ef52520693e8e811a714c7b20" +dependencies = [ + "Inflector", + "pmutil", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] @@ -392,41 +205,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] -name = "lalrpop" -version = "0.17.2" +name = "lalrpop-util" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64dc3698e75d452867d9bd86f4a723f452ce9d01fe1d55990b79f0c790aa67db" +checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d" + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" dependencies = [ - "ascii-canvas", - "atty", - "bit-set", - "diff", - "docopt", - "ena", - "itertools", - "lalrpop-util", - "petgraph", - "regex", - "regex-syntax 0.6.29", - "serde", - "serde_derive", - "sha2", - "string_cache", - "term", - "unicode-xid 0.1.0", + "lexical-parse-integer", + "lexical-util", + "static_assertions", ] [[package]] -name = "lalrpop-util" -version = "0.17.2" +name = "lexical-parse-integer" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c277d18683b36349ab5cd030158b54856fca6bb2d5dc5263b06288f486958b7c" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] [[package]] -name = "lazy_static" -version = "1.4.0" +name = "lexical-util" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] [[package]] name = "libc" @@ -456,6 +268,12 @@ dependencies = [ "url", ] +[[package]] +name = "matches" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" + [[package]] name = "memchr" version = "2.5.0" @@ -468,7 +286,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ - "autocfg 1.1.0", + "autocfg", ] [[package]] @@ -480,12 +298,6 @@ dependencies = [ "adler", ] -[[package]] -name = "new_debug_unreachable" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" - [[package]] name = "nix" version = "0.23.2" @@ -501,11 +313,11 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.2.6" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-integer", "num-traits", ] @@ -516,7 +328,7 @@ version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-traits", ] @@ -526,7 +338,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ - "autocfg 1.1.0", + "autocfg", ] [[package]] @@ -539,16 +351,10 @@ dependencies = [ ] [[package]] -name = "opaque-debug" -version = "0.2.3" +name = "once_cell" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - -[[package]] -name = "ordermap" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "percent-encoding" @@ -557,20 +363,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] -name = "petgraph" -version = "0.4.13" +name = "phf" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" +checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" dependencies = [ - "fixedbitset", - "ordermap", + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" +dependencies = [ + "phf_generator", + "phf_shared", ] [[package]] name = "phf_generator" -version = "0.7.24" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" dependencies = [ "phf_shared", "rand", @@ -578,18 +393,23 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.7.24" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" +checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" dependencies = [ "siphasher", ] [[package]] -name = "precomputed-hash" -version = "0.1.1" +name = "pmutil" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +checksum = "3894e5d549cccbe44afecf72922f277f603cd4bb0219c8342631ef18fffbe004" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] [[package]] name = "proc-macro2" @@ -606,6 +426,7 @@ version = "0.0.29" dependencies = [ "erg_common", "erg_compiler", + "rustpython-ast", "rustpython-parser", ] @@ -631,170 +452,31 @@ dependencies = [ [[package]] name = "rand" -version = "0.6.5" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "autocfg 0.1.8", - "libc", - "rand_chacha", - "rand_core 0.4.2", - "rand_hc", - "rand_isaac", - "rand_jitter", - "rand_os", - "rand_pcg", - "rand_xorshift", - "winapi", -] - -[[package]] -name = "rand_chacha" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.3.1", + "rand_core", ] [[package]] name = "rand_core" -version = "0.3.1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "ruff_source_location" +version = "0.0.0" dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - -[[package]] -name = "rand_hc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_isaac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_jitter" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -dependencies = [ - "libc", - "rand_core 0.4.2", - "winapi", -] - -[[package]] -name = "rand_os" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -dependencies = [ - "cloudabi", - "fuchsia-cprng", - "libc", - "rand_core 0.4.2", - "rdrand", - "winapi", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.4.2", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "redox_syscall" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - -[[package]] -name = "redox_users" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" -dependencies = [ - "getrandom", - "redox_syscall", - "rust-argon2", -] - -[[package]] -name = "regex" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" -dependencies = [ - "aho-corasick", "memchr", - "regex-syntax 0.7.1", + "once_cell", + "ruff_text_size", ] [[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - -[[package]] -name = "regex-syntax" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" - -[[package]] -name = "rust-argon2" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" -dependencies = [ - "base64", - "blake2b_simd", - "constant_time_eq", - "crossbeam-utils", -] +name = "ruff_text_size" +version = "0.0.0" [[package]] name = "rustc-demangle" @@ -803,21 +485,64 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] -name = "rustpython-parser" -version = "0.1.2" +name = "rustc-hash" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85b1038ecd8791bdae455ae784b48f522ebeca1b3323d0af2b251c5c8ff1c68" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustpython-ast" +version = "0.2.0" dependencies = [ - "lalrpop", + "is-macro", + "num-bigint", + "rustpython-literal", + "rustpython-parser-core", + "static_assertions", +] + +[[package]] +name = "rustpython-literal" +version = "0.2.0" +dependencies = [ + "hexf-parse", + "is-macro", + "lexical-parse-float", + "num-traits", + "unic-ucd-category", +] + +[[package]] +name = "rustpython-parser" +version = "0.2.0" +dependencies = [ + "anyhow", + "is-macro", + "itertools", "lalrpop-util", "log", "num-bigint", "num-traits", + "phf", + "phf_codegen", + "rustc-hash", + "rustpython-ast", + "rustpython-parser-core", + "tiny-keccak", "unic-emoji-char", "unic-ucd-ident", "unicode_names2", ] +[[package]] +name = "rustpython-parser-core" +version = "0.2.0" +dependencies = [ + "is-macro", + "ruff_source_location", + "ruff_text_size", +] + [[package]] name = "ryu" version = "1.0.13" @@ -841,7 +566,7 @@ checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.16", ] [[package]] @@ -863,67 +588,32 @@ checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" dependencies = [ "proc-macro2", "quote", - "syn", -] - -[[package]] -name = "sha2" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" -dependencies = [ - "block-buffer", - "digest", - "fake-simd", - "opaque-debug", + "syn 2.0.16", ] [[package]] name = "siphasher" -version = "0.2.3" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" [[package]] -name = "string_cache" -version = "0.7.5" +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89c058a82f9fd69b1becf8c274f412281038877c553182f1d02eb027045a2d67" -dependencies = [ - "lazy_static", - "new_debug_unreachable", - "phf_shared", - "precomputed-hash", - "serde", - "string_cache_codegen", - "string_cache_shared", -] +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] -name = "string_cache_codegen" -version = "0.4.4" +name = "syn" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "phf_generator", - "phf_shared", "proc-macro2", "quote", - "string_cache_shared", + "unicode-ident", ] -[[package]] -name = "string_cache_shared" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "syn" version = "2.0.16" @@ -936,14 +626,12 @@ dependencies = [ ] [[package]] -name = "term" -version = "0.5.2" +name = "tiny-keccak" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" dependencies = [ - "byteorder", - "dirs", - "winapi", + "crunchy", ] [[package]] @@ -961,12 +649,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" -[[package]] -name = "typenum" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" - [[package]] name = "unic-char-property" version = "0.9.0" @@ -999,6 +681,18 @@ dependencies = [ "unic-ucd-version", ] +[[package]] +name = "unic-ucd-category" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8d4591f5fcfe1bd4453baaf803c40e1b1e69ff8455c47620440b46efef91c0" +dependencies = [ + "matches", + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + [[package]] name = "unic-ucd-ident" version = "0.9.0" @@ -1040,12 +734,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - [[package]] name = "unicode-xid" version = "0.2.4" @@ -1054,9 +742,11 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "unicode_names2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87d6678d7916394abad0d4b19df4d3802e1fd84abd7d701f39b75ee71b9e8cf1" +version = "0.6.0" +source = "git+https://github.com/youknowone/unicode_names2.git?rev=4ce16aa85cbcdd9cc830410f1a72ef9a235f2fde#4ce16aa85cbcdd9cc830410f1a72ef9a235f2fde" +dependencies = [ + "phf", +] [[package]] name = "url" @@ -1070,12 +760,6 @@ dependencies = [ "serde", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 74e0a4d..c228898 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,9 @@ repository = "https://github.com/mtshiba/pylyzer" erg_common = { version = "0.6.14-nightly.0", features = ["py_compat", "els"] } erg_compiler = { version = "0.6.14-nightly.0", features = ["py_compat", "els"] } els = { version = "0.1.26-nightly.0", features = ["py_compat"] } -rustpython-parser = "0.1.2" +# rustpython-parser = { git = "https://github.com/RustPython/Parser.git", rev = "4de0cb18277f45aecd1cfdcc5ac69ab6bd9e2854", features = ["rustpython-literal"] } +rustpython-ast = { path = "../RustPython-Parser/ast", features = ["rustpython-literal"] } +rustpython-parser = { path = "../RustPython-Parser/parser" } # erg_compiler = { git = "https://github.com/erg-lang/erg", branch = "main", features = ["py_compat", "els"] } # erg_common = { git = "https://github.com/erg-lang/erg", branch = "main", features = ["py_compat", "els"] } # els = { git = "https://github.com/erg-lang/erg", branch = "main", features = ["py_compat"] } diff --git a/crates/py2erg/Cargo.toml b/crates/py2erg/Cargo.toml index 77471b6..94461c3 100644 --- a/crates/py2erg/Cargo.toml +++ b/crates/py2erg/Cargo.toml @@ -14,6 +14,7 @@ debug = ["erg_compiler/debug", "erg_common/debug"] [dependencies] rustpython-parser = { workspace = true } +rustpython-ast = { workspace = true } erg_common = { workspace = true } erg_compiler = { workspace = true } diff --git a/crates/py2erg/ast_util.rs b/crates/py2erg/ast_util.rs index 8d9d87e..d784758 100644 --- a/crates/py2erg/ast_util.rs +++ b/crates/py2erg/ast_util.rs @@ -1,124 +1,12 @@ use erg_common::log; -use rustpython_parser::ast::{ - BooleanOperator, Comparison, ExpressionType, Keyword, Number, StringGroup, -}; +use rustpython_parser::ast::{Boolop, Cmpop, Constant, ExpressionType, Keyword, StringGroup}; -pub fn number_to_string(num: &Number) -> String { - match num { - Number::Integer { value } => value.to_string(), - Number::Float { value } => value.to_string(), - Number::Complex { real, imag } => format!("{real}+{imag}j"), - } +pub fn number_to_string(num: &Constant) -> String { + num.to_string() } -pub fn keyword_length(keyword: &Keyword) -> usize { - if let Some(name) = &keyword.name { - name.len() + 1 + length(&keyword.value.node) - } else { - length(&keyword.value.node) - } -} - -pub fn string_length(string: &StringGroup) -> usize { - match string { - StringGroup::Constant { value } => value.len(), - StringGroup::Joined { values } => values.iter().map(string_length).sum(), - other => { - log!(err "{other:?}"); - 0 - } - } -} - -pub fn comp_to_string(comp: &Comparison) -> String { - match comp { - Comparison::In => "in".to_string(), - Comparison::NotIn => "not in".to_string(), - Comparison::Is => "is".to_string(), - Comparison::IsNot => "is not".to_string(), - Comparison::Less => "<".to_string(), - Comparison::Greater => ">".to_string(), - Comparison::Equal => "==".to_string(), - Comparison::NotEqual => "!=".to_string(), - Comparison::LessOrEqual => "<=".to_string(), - Comparison::GreaterOrEqual => ">=".to_string(), - } -} - -pub fn length(expr: &ExpressionType) -> usize { - match expr { - ExpressionType::Identifier { name } => name.len(), - ExpressionType::Number { value } => number_to_string(value).len(), - ExpressionType::String { value } => string_length(value), - ExpressionType::Attribute { value, name } => length(&value.node) + name.len() + 1, - ExpressionType::Subscript { a, b } => length(&a.node) + length(&b.node) + 2, - ExpressionType::Tuple { elements } - | ExpressionType::List { elements } - | ExpressionType::Set { elements } => { - if let (Some(first), Some(last)) = (elements.first(), elements.last()) { - 2 + last.location.column() - first.location.column() - } else { - 2 - } - } - ExpressionType::Call { - function, - args, - keywords, - } => { - let args_len = if let (Some(first), Some(last)) = (args.first(), args.last()) { - last.location.column() - first.location.column() - } else { - 0 - }; - let kw_len = if let (Some(first), Some(last)) = (keywords.first(), keywords.last()) { - last.value.location.column() - first.value.location.column() - } else { - 0 - }; - length(&function.node) + args_len + kw_len + 2 // () - } - ExpressionType::Unop { op: _, a } => 1 + length(&a.node), - ExpressionType::Binop { a, op: _, b } => length(&a.node) + 3 + length(&b.node), - ExpressionType::BoolOp { op, values } => match op { - BooleanOperator::And => values - .iter() - .map(|elem| length(&elem.node)) - .fold(0, |acc, x| acc + x + 3), - BooleanOperator::Or => values - .iter() - .map(|elem| length(&elem.node)) - .fold(0, |acc, x| acc + x + 2), - }, - ExpressionType::Compare { vals, ops } => vals - .iter() - .zip(ops.iter()) - .map(|(elem, op)| length(&elem.node) + comp_to_string(op).len()) - .fold(0, |acc, x| acc + x + 2), - ExpressionType::IfExpression { test, body, orelse } => { - // x if y else z - length(&test.node) + 4 + length(&body.node) + 6 + length(&orelse.node) - } - ExpressionType::Lambda { args: _, body } => { - // lambda x: y - // TODO: - 7 + 1 + length(&body.node) - } - ExpressionType::Await { value } => 5 + length(&value.node), - ExpressionType::Yield { value } => 5 + value.as_ref().map(|x| length(&x.node)).unwrap_or(0), - ExpressionType::NamedExpression { left, right } => { - // x := y - length(&left.node) + 4 + length(&right.node) - } - ExpressionType::Starred { value } => 1 + length(&value.node), - ExpressionType::False => 5, - ExpressionType::True | ExpressionType::None => 4, - ExpressionType::Ellipsis => 8, - other => { - log!(err "{other:?}"); - 0 - } - } +pub fn comp_to_string(comp: &Cmpop) -> String { + cmpop.as_str() } pub fn accessor_name(expr: ExpressionType) -> Option { diff --git a/crates/py2erg/clone.rs b/crates/py2erg/clone.rs deleted file mode 100644 index b02daa0..0000000 --- a/crates/py2erg/clone.rs +++ /dev/null @@ -1,272 +0,0 @@ -use rustpython_parser::ast::{ - BooleanOperator, Comparison, Comprehension, ComprehensionKind, ExpressionType, Keyword, - Located, Number, Operator, Parameter, Parameters, StringGroup, UnaryOperator, Varargs, -}; - -fn clone_number(num: &Number) -> Number { - match num { - Number::Integer { value } => Number::Integer { - value: value.clone(), - }, - Number::Float { value } => Number::Float { value: *value }, - Number::Complex { real, imag } => Number::Complex { - real: *real, - imag: *imag, - }, - } -} - -fn clone_string_group(group: &StringGroup) -> StringGroup { - match group { - StringGroup::Constant { value } => StringGroup::Constant { - value: value.clone(), - }, - StringGroup::FormattedValue { - value, - conversion, - spec, - } => StringGroup::FormattedValue { - value: Box::new(clone_loc_expr(value)), - conversion: *conversion, - spec: spec.as_deref().map(|sp| Box::new(clone_string_group(sp))), - }, - StringGroup::Joined { values } => StringGroup::Joined { - values: values.iter().map(clone_string_group).collect::>(), - }, - } -} - -fn clone_unary_op(op: &UnaryOperator) -> UnaryOperator { - match op { - UnaryOperator::Not => UnaryOperator::Not, - UnaryOperator::Inv => UnaryOperator::Inv, - UnaryOperator::Pos => UnaryOperator::Pos, - UnaryOperator::Neg => UnaryOperator::Neg, - } -} - -fn clone_bin_op(op: &Operator) -> Operator { - match op { - Operator::Add => Operator::Add, - Operator::Sub => Operator::Sub, - Operator::Mult => Operator::Mult, - Operator::MatMult => Operator::MatMult, - Operator::Div => Operator::Div, - Operator::Mod => Operator::Mod, - Operator::Pow => Operator::Pow, - Operator::LShift => Operator::LShift, - Operator::RShift => Operator::RShift, - Operator::BitOr => Operator::BitOr, - Operator::BitXor => Operator::BitXor, - Operator::BitAnd => Operator::BitAnd, - Operator::FloorDiv => Operator::FloorDiv, - } -} - -fn clone_comp_op(op: &Comparison) -> Comparison { - match op { - Comparison::Equal => Comparison::Equal, - Comparison::NotEqual => Comparison::NotEqual, - Comparison::Less => Comparison::Less, - Comparison::LessOrEqual => Comparison::LessOrEqual, - Comparison::Greater => Comparison::Greater, - Comparison::GreaterOrEqual => Comparison::GreaterOrEqual, - Comparison::Is => Comparison::Is, - Comparison::IsNot => Comparison::IsNot, - Comparison::In => Comparison::In, - Comparison::NotIn => Comparison::NotIn, - } -} - -fn clone_bool_op(op: &BooleanOperator) -> BooleanOperator { - match op { - BooleanOperator::And => BooleanOperator::And, - BooleanOperator::Or => BooleanOperator::Or, - } -} - -fn clone_param(param: &Parameter) -> Parameter { - Parameter { - location: param.location, - arg: param.arg.clone(), - annotation: param - .annotation - .as_deref() - .map(|a| Box::new(clone_loc_expr(a))), - } -} - -fn clone_varargs(varargs: &Varargs) -> Varargs { - match varargs { - Varargs::None => Varargs::None, - Varargs::Unnamed => Varargs::Unnamed, - Varargs::Named(name) => Varargs::Named(clone_param(name)), - } -} - -fn clone_params(params: &Parameters) -> Parameters { - Parameters { - posonlyargs_count: params.posonlyargs_count, - args: params.args.iter().map(clone_param).collect::>(), - vararg: clone_varargs(¶ms.vararg), - kwonlyargs: params - .kwonlyargs - .iter() - .map(clone_param) - .collect::>(), - kw_defaults: params - .kw_defaults - .iter() - .map(|def| def.as_ref().map(clone_loc_expr)) - .collect::>(), - kwarg: clone_varargs(¶ms.kwarg), - defaults: params - .defaults - .iter() - .map(clone_loc_expr) - .collect::>(), - } -} - -fn clone_kw(keyword: &Keyword) -> Keyword { - Keyword { - name: keyword.name.clone(), - value: clone_loc_expr(&keyword.value), - } -} - -fn clone_comprehension_kind(kind: &ComprehensionKind) -> ComprehensionKind { - match kind { - ComprehensionKind::Dict { key, value } => ComprehensionKind::Dict { - key: clone_loc_expr(key), - value: clone_loc_expr(value), - }, - ComprehensionKind::List { element } => ComprehensionKind::List { - element: clone_loc_expr(element), - }, - ComprehensionKind::Set { element } => ComprehensionKind::Set { - element: clone_loc_expr(element), - }, - ComprehensionKind::GeneratorExpression { element } => { - ComprehensionKind::GeneratorExpression { - element: clone_loc_expr(element), - } - } - } -} - -pub fn clone_loc_expr(expr: &Located) -> Located { - Located { - node: clone_expr(&expr.node), - location: expr.location, - } -} - -pub fn clone_expr(expr: &ExpressionType) -> ExpressionType { - match expr { - ExpressionType::None => ExpressionType::None, - ExpressionType::Ellipsis => ExpressionType::Ellipsis, - ExpressionType::True => ExpressionType::True, - ExpressionType::False => ExpressionType::False, - ExpressionType::Identifier { name } => ExpressionType::Identifier { name: name.clone() }, - ExpressionType::Number { value } => ExpressionType::Number { - value: clone_number(value), - }, - ExpressionType::String { value } => ExpressionType::String { - value: clone_string_group(value), - }, - ExpressionType::Attribute { value, name } => ExpressionType::Attribute { - value: Box::new(clone_loc_expr(value)), - name: name.clone(), - }, - ExpressionType::Subscript { a, b } => ExpressionType::Subscript { - a: Box::new(clone_loc_expr(a)), - b: Box::new(clone_loc_expr(b)), - }, - ExpressionType::Slice { elements } => ExpressionType::Slice { - elements: elements.iter().map(clone_loc_expr).collect::>(), - }, - ExpressionType::Bytes { value } => ExpressionType::Bytes { - value: value.clone(), - }, - ExpressionType::Call { - function, - args, - keywords, - } => ExpressionType::Call { - function: Box::new(clone_loc_expr(function)), - args: args.iter().map(clone_loc_expr).collect::>(), - keywords: keywords.iter().map(clone_kw).collect::>(), - }, - ExpressionType::Unop { op, a } => ExpressionType::Unop { - op: clone_unary_op(op), - a: Box::new(clone_loc_expr(a)), - }, - ExpressionType::Binop { a, op, b } => ExpressionType::Binop { - a: Box::new(clone_loc_expr(a)), - op: clone_bin_op(op), - b: Box::new(clone_loc_expr(b)), - }, - ExpressionType::Compare { vals, ops } => ExpressionType::Compare { - ops: ops.iter().map(clone_comp_op).collect::>(), - vals: vals.iter().map(clone_loc_expr).collect::>(), - }, - ExpressionType::BoolOp { op, values } => ExpressionType::BoolOp { - op: clone_bool_op(op), - values: values.iter().map(clone_loc_expr).collect::>(), - }, - ExpressionType::Lambda { args, body } => ExpressionType::Lambda { - args: Box::new(clone_params(args)), - body: Box::new(clone_loc_expr(body)), - }, - ExpressionType::IfExpression { test, body, orelse } => ExpressionType::IfExpression { - test: Box::new(clone_loc_expr(test)), - body: Box::new(clone_loc_expr(body)), - orelse: Box::new(clone_loc_expr(orelse)), - }, - ExpressionType::Dict { elements } => ExpressionType::Dict { - elements: elements - .iter() - .map(|(key, value)| (key.as_ref().map(clone_loc_expr), clone_loc_expr(value))) - .collect::>(), - }, - ExpressionType::Set { elements } => ExpressionType::Set { - elements: elements.iter().map(clone_loc_expr).collect::>(), - }, - ExpressionType::List { elements } => ExpressionType::List { - elements: elements.iter().map(clone_loc_expr).collect::>(), - }, - ExpressionType::Tuple { elements } => ExpressionType::Tuple { - elements: elements.iter().map(clone_loc_expr).collect::>(), - }, - ExpressionType::Yield { value } => ExpressionType::Yield { - value: value.as_ref().map(|val| Box::new(clone_loc_expr(val))), - }, - ExpressionType::YieldFrom { value } => ExpressionType::YieldFrom { - value: Box::new(clone_loc_expr(value)), - }, - ExpressionType::Await { value } => ExpressionType::Await { - value: Box::new(clone_loc_expr(value)), - }, - ExpressionType::NamedExpression { left, right } => ExpressionType::NamedExpression { - left: Box::new(clone_loc_expr(left)), - right: Box::new(clone_loc_expr(right)), - }, - ExpressionType::Starred { value } => ExpressionType::Starred { - value: Box::new(clone_loc_expr(value)), - }, - ExpressionType::Comprehension { kind, generators } => ExpressionType::Comprehension { - kind: Box::new(clone_comprehension_kind(kind)), - generators: generators - .iter() - .map(|gen| Comprehension { - location: gen.location, - target: clone_loc_expr(&gen.target), - iter: clone_loc_expr(&gen.iter), - ifs: gen.ifs.iter().map(clone_loc_expr).collect::>(), - is_async: gen.is_async, - }) - .collect::>(), - }, - } -} diff --git a/crates/py2erg/convert.rs b/crates/py2erg/convert.rs index 519e700..ec2d589 100644 --- a/crates/py2erg/convert.rs +++ b/crates/py2erg/convert.rs @@ -22,14 +22,15 @@ use erg_compiler::erg_parser::desugar::Desugarer; use erg_compiler::erg_parser::token::{Token, TokenKind, AS, DOT, EQUAL}; use erg_compiler::erg_parser::Parser; use erg_compiler::error::{CompileError, CompileErrors}; -use rustpython_parser::ast::{ - BooleanOperator, Comparison, ExpressionType, ImportSymbol, Located, Number, Operator, - Parameter, Parameters, Program, StatementType, StringGroup, Suite, UnaryOperator, +use rustpython_parser::ast::located::{ + self as py_ast, Boolop, Cmpop, ImportSymbol, Keyword, Located, ModModule, Number, Operator, + Parameter, Parameters, StringGroup, Suite, Suite, Unaryop, +}; +use rustpython_parser::source_code::{ + SourceLocation as PyLocation, SourceLocator, SourceRange as PySourceRange, }; -use rustpython_parser::ast::{Keyword, Location as PyLocation}; -use crate::ast_util::{accessor_name, length}; -use crate::clone::clone_loc_expr; +use crate::ast_util::accessor_name; use crate::error::*; pub const ARROW: Token = Token::dummy(TokenKind::FuncArrow, "->"); @@ -117,12 +118,12 @@ fn op_to_token(op: Operator) -> Token { Token::from_str(kind, cont) } -pub fn pyloc_to_ergloc(loc: PyLocation, cont_len: usize) -> erg_common::error::Location { +pub fn pyloc_to_ergloc(range: PySourceRange) -> erg_common::error::Location { erg_common::error::Location::range( - loc.row() as u32, - loc.column() as u32, - loc.row() as u32, - (loc.column() + cont_len) as u32, + loc.start.row() as u32, + loc.start.column() as u32, + loc.end.row() as u32, + loc.end.column() as u32, ) } @@ -369,7 +370,7 @@ impl ASTConverter { .annotation .map(|anot| { ( - self.convert_type_spec(clone_loc_expr(&anot)), + self.convert_type_spec(anot.clone()), self.convert_expr(*anot), ) }) @@ -380,7 +381,7 @@ impl ASTConverter { fn convert_default_param( &mut self, kw: Parameter, - default: Located, + default: py_ast::Expr, ) -> DefaultParamSignature { let sig = self.convert_nd_param(kw); let default = self.convert_expr(default); @@ -422,7 +423,7 @@ impl ASTConverter { /// (i, j) => $1 (i = $1[0]; j = $1[1]) fn convert_opt_expr_to_param( &mut self, - expr: Option>, + expr: Option, ) -> (NonDefaultParamSignature, Vec) { if let Some(expr) = expr { self.convert_expr_to_param(expr) @@ -437,22 +438,23 @@ impl ASTConverter { fn convert_expr_to_param( &mut self, - expr: Located, + expr: py_ast::Expr, ) -> (NonDefaultParamSignature, Vec) { - match expr.node { - ExpressionType::Identifier { name } => { - (self.convert_for_param(name, expr.location), vec![]) - } - ExpressionType::Tuple { elements } => { + match expr { + py_ast::Expr::Name(expr) => ( + self.convert_for_param(expr.name, expr.location().start()), + vec![], + ), + py_ast::Expr::Tuple(expr) => { let tmp = fresh_varname(); let tmp_name = - VarName::from_str_and_line((&tmp).into(), expr.location.row() as u32); + VarName::from_str_and_line((&tmp).into(), expr.location().start().row() as u32); let tmp_expr = Expr::Accessor(Accessor::Ident(Identifier::new( VisModifierSpec::Public(DOT), tmp_name.clone(), ))); let mut block = vec![]; - for (i, elem) in elements.into_iter().enumerate() { + for (i, elem) in expr.elts.into_iter().enumerate() { let index = Literal::new(Token::new( TokenKind::NatLit, i.to_string(), @@ -464,9 +466,9 @@ impl ASTConverter { Self::param_pattern_to_var(param.pat), param.t_spec, )); - let method = tmp_expr - .clone() - .attr_expr(self.convert_ident("__getitem__".to_string(), expr.location)); + let method = tmp_expr.clone().attr_expr( + self.convert_ident("__getitem__".to_string(), expr.location().start()), + ); let tuple_acc = method.call1(Expr::Literal(index)); let body = DefBody::new(EQUAL, Block::new(vec![tuple_acc]), DefId(0)); let def = Expr::Def(Def::new(sig, body)); @@ -480,8 +482,8 @@ impl ASTConverter { let token = Token::new( TokenKind::UBar, "_", - expr.location.row() as u32, - expr.location.column() as u32 - 1, + expr.location().start().row() as u32, + expr.location().start().column() as u32 - 1, ); ( NonDefaultParamSignature::new(ParamPattern::Discard(token), None), @@ -491,7 +493,7 @@ impl ASTConverter { } } - fn convert_for_body(&mut self, lhs: Option>, body: Suite) -> Lambda { + fn convert_for_body(&mut self, lhs: Option, body: Suite) -> Lambda { let (param, block) = self.convert_opt_expr_to_param(lhs); let params = Params::new(vec![param], None, vec![], None); self.block_id_counter += 1; @@ -521,18 +523,14 @@ impl ASTConverter { } // TODO: - fn convert_compound_type_spec( - &mut self, - name: String, - args: Located, - ) -> TypeSpec { + fn convert_compound_type_spec(&mut self, name: String, args: py_ast::Expr) -> TypeSpec { match &name[..] { "Union" => { - let ExpressionType::Tuple { mut elements } = args.node else { + let py_ast::Expr::Tuple { mut elements } = args.node else { let err = CompileError::syntax_error( self.cfg.input.clone(), line!() as usize, - pyloc_to_ergloc(args.location, length(&args.node)), + pyloc_to_ergloc(args.range()), self.cur_namespace(), "`Union` takes at least 2 types".into(), None, @@ -557,7 +555,7 @@ impl ASTConverter { TypeSpec::or(t, none) } "Literal" => { - let ExpressionType::Tuple { elements } = args.node else { + let py_ast::Expr::Tuple { elements } = args.node else { return Self::gen_dummy_type_spec(args.location); }; let mut elems = vec![]; @@ -582,13 +580,13 @@ impl ASTConverter { } // TODO: distinguish from collections.abc.Callable "Callable" => { - let ExpressionType::Tuple { mut elements } = args.node else { + let py_ast::Expr::Tuple { mut elements } = args.node else { return Self::gen_dummy_type_spec(args.location); }; let params = elements.remove(0); let mut non_defaults = vec![]; match params.node { - ExpressionType::List { elements } => { + py_ast::Expr::List { elements } => { for param in elements.into_iter() { let t_spec = self.convert_type_spec(param); non_defaults.push(ParamTySpec::anonymous(t_spec)); @@ -598,7 +596,7 @@ impl ASTConverter { let err = CompileError::syntax_error( self.cfg.input.clone(), line!() as usize, - pyloc_to_ergloc(params.location, length(&other)), + pyloc_to_ergloc(params.range()), self.cur_namespace(), "Expected a list of parameters".into(), None, @@ -642,11 +640,11 @@ impl ASTConverter { TypeSpec::poly(acc, ConstArgs::pos_only(vec![elem_t], None)) } "Mapping" | "MutableMapping" => { - let ExpressionType::Tuple { mut elements } = args.node else { + let py_ast::Expr::Tuple { mut elements } = args.node else { let err = CompileError::syntax_error( self.cfg.input.clone(), line!() as usize, - pyloc_to_ergloc(args.location, length(&args.node)), + pyloc_to_ergloc(args.location.range()), self.cur_namespace(), format!("`{name}` takes 2 types"), None, @@ -718,7 +716,7 @@ impl ASTConverter { TypeSpec::poly(acc, ConstArgs::new(vec![elem_t, len], None, vec![], None)) } "dict" => { - let ExpressionType::Tuple { mut elements } = args.node else { + let py_ast::Expr::Tuple { mut elements } = args.node else { return Self::gen_dummy_type_spec(args.location); }; let (l_brace, r_brace) = @@ -763,7 +761,7 @@ impl ASTConverter { TypeSpec::poly(acc, ConstArgs::new(vec![dict], None, vec![], None)) } "tuple" => { - let ExpressionType::Tuple { elements } = args.node else { + let py_ast::Expr::Tuple { elements } = args.node else { return Self::gen_dummy_type_spec(args.location); }; let parens = @@ -779,22 +777,24 @@ impl ASTConverter { } } - fn convert_type_spec(&mut self, expr: Located) -> TypeSpec { + fn convert_type_spec(&mut self, expr: py_ast::Expr) -> TypeSpec { #[allow(clippy::collapsible_match)] match expr.node { - ExpressionType::Identifier { name } => { - self.convert_ident_type_spec(name, expr.location) + py_ast::Expr::Identifier { name } => { + self.convert_ident_type_spec(name, expr.location().start()) } - ExpressionType::None => self.convert_ident_type_spec("NoneType".into(), expr.location), - ExpressionType::Attribute { value, name } => { + py_ast::Expr::None => { + self.convert_ident_type_spec("NoneType".into(), expr.location().start()) + } + py_ast::Expr::Attribute { value, name } => { let namespace = Box::new(self.convert_expr(*value)); - let t = self.convert_ident(name, expr.location); + let t = self.convert_ident(name, expr.location().start()); let predecl = PreDeclTypeSpec::Attr { namespace, t }; TypeSpec::PreDeclTy(predecl) } - ExpressionType::Subscript { a, b } => match a.node { - ExpressionType::Identifier { name } => self.convert_compound_type_spec(name, *b), - ExpressionType::Attribute { value, name } => { + py_ast::Expr::Subscript { a, b } => match a.node { + py_ast::Expr::Identifier { name } => self.convert_compound_type_spec(name, *b), + py_ast::Expr::Attribute { value, name } => { match accessor_name(value.node).as_ref().map(|s| &s[..]) { Some("typing" | "collections.abc") => { self.convert_compound_type_spec(name, *b) @@ -810,7 +810,7 @@ impl ASTConverter { Self::gen_dummy_type_spec(a.location) } }, - ExpressionType::Binop { a, op, b } => { + py_ast::Expr::Binop { a, op, b } => { match op { // A | B Operator::BitOr => { @@ -818,12 +818,12 @@ impl ASTConverter { let rhs = self.convert_type_spec(*b); TypeSpec::or(lhs, rhs) } - _ => Self::gen_dummy_type_spec(expr.location), + _ => Self::gen_dummy_type_spec(expr.location().start()), } } other => { log!(err "unknown: {other:?}"); - Self::gen_dummy_type_spec(expr.location) + Self::gen_dummy_type_spec(expr.location().start()) } } } @@ -834,7 +834,7 @@ impl ASTConverter { expr_loc: PyLocation, ) -> (Token, Token) where - Elems: Iterator> + ExactSizeIterator, + Elems: Iterator + ExactSizeIterator, { let (l_cont, r_cont, r_kind) = match l_kind { TokenKind::LBrace => ("{", "}", TokenKind::RBrace), @@ -868,9 +868,9 @@ impl ASTConverter { Expr::UnaryOp(UnaryOp::new(mut_op, expr)) } - fn convert_expr(&mut self, expr: Located) -> Expr { + fn convert_expr(&mut self, expr: py_ast::Expr) -> Expr { match expr.node { - ExpressionType::Number { value } => { + py_ast::Expr::Number { value } => { let (kind, cont) = match value { Number::Integer { value } if value >= 0.into() => { (TokenKind::NatLit, value.to_string()) @@ -884,12 +884,12 @@ impl ASTConverter { let token = Token::new( kind, cont, - expr.location.row() as u32, - expr.location.column() as u32 - 1, + expr.location().start().row() as u32, + expr.location().start().column() as u32 - 1, ); Expr::Literal(Literal::new(token)) } - ExpressionType::String { value } => { + py_ast::Expr::String { value } => { let StringGroup::Constant{ value } = value else { return Expr::Dummy(Dummy::new(None, vec![])); }; @@ -898,40 +898,40 @@ impl ASTConverter { let token = Token::new( TokenKind::StrLit, value, - expr.location.row() as u32, - expr.location.column() as u32 - 2, + expr.location().start().row() as u32, + expr.location().start().column() as u32 - 2, ); Expr::Literal(Literal::new(token)) } - ExpressionType::False => Expr::Literal(Literal::new(Token::new( + py_ast::Expr::False => Expr::Literal(Literal::new(Token::new( TokenKind::BoolLit, "False", - expr.location.row() as u32, - expr.location.column() as u32 - 1, + expr.location().start().row() as u32, + expr.location().start().column() as u32 - 1, ))), - ExpressionType::True => Expr::Literal(Literal::new(Token::new( + py_ast::Expr::True => Expr::Literal(Literal::new(Token::new( TokenKind::BoolLit, "True", - expr.location.row() as u32, - expr.location.column() as u32 - 1, + expr.location().start().row() as u32, + expr.location().start().column() as u32 - 1, ))), - ExpressionType::None => Expr::Literal(Literal::new(Token::new( + py_ast::Expr::None => Expr::Literal(Literal::new(Token::new( TokenKind::NoneLit, "None", - expr.location.row() as u32, - expr.location.column() as u32 - 1, + expr.location().start().row() as u32, + expr.location().start().column() as u32 - 1, ))), - ExpressionType::Ellipsis => Expr::Literal(Literal::new(Token::new( + py_ast::Expr::Ellipsis => Expr::Literal(Literal::new(Token::new( TokenKind::EllipsisLit, "...", - expr.location.row() as u32, - expr.location.column() as u32 - 1, + expr.location().start().row() as u32, + expr.location().start().column() as u32 - 1, ))), - ExpressionType::Identifier { name } => { - let ident = self.convert_ident(name, expr.location); + py_ast::Expr::Identifier { name } => { + let ident = self.convert_ident(name, expr.location().start()); Expr::Accessor(Accessor::Ident(ident)) } - ExpressionType::Attribute { value, name } => { + py_ast::Expr::Attribute { value, name } => { let obj = self.convert_expr(*value); let attr_name_loc = PyLocation::new( obj.ln_end().unwrap_or(1) as usize, @@ -940,13 +940,13 @@ impl ASTConverter { let name = self.convert_attr_ident(name, attr_name_loc); obj.attr_expr(name) } - ExpressionType::IfExpression { test, body, orelse } => { + py_ast::Expr::IfExpression { test, body, orelse } => { let block = self.convert_expr(*body); let params = Params::new(vec![], None, vec![], None); let sig = LambdaSignature::new(params.clone(), None, TypeBoundSpecs::empty()); let body = Lambda::new(sig, Token::DUMMY, Block::new(vec![block]), DefId(0)); let test = self.convert_expr(*test); - let if_ident = self.convert_ident("if".to_string(), expr.location); + let if_ident = self.convert_ident("if".to_string(), expr.location().start()); let if_acc = Expr::Accessor(Accessor::Ident(if_ident)); let else_block = self.convert_expr(*orelse); let sig = LambdaSignature::new(params, None, TypeBoundSpecs::empty()); @@ -962,7 +962,7 @@ impl ASTConverter { ); if_acc.call_expr(args) } - ExpressionType::Call { + py_ast::Expr::Call { function, args, keywords, @@ -974,12 +974,9 @@ impl ASTConverter { .collect::>(); let kw_args = keywords .into_iter() - .map(|Keyword { name, value }| { - let name = name.unwrap_or_default(); - let name = Token::symbol_with_loc( - &name, - pyloc_to_ergloc(value.location, name.len()), - ); + .map(|Keyword { arg, value, range }| { + let name = arg.unwrap_or_default(); + let name = Token::symbol_with_loc(name.as_str(), pyloc_to_ergloc(range)); let ex = self.convert_expr(value); KwArg::new(name, None, ex) }) @@ -992,13 +989,13 @@ impl ASTConverter { let lp = Token::new( TokenKind::LParen, "(", - expr.location.row() as u32, + expr.location().start().row() as u32, function.col_end().unwrap_or(0) + 1, ); let rp = Token::new( TokenKind::RParen, ")", - expr.location.row() as u32, + expr.location().start().row() as u32, last_col + 1, ); (lp, rp) @@ -1006,55 +1003,55 @@ impl ASTConverter { let args = Args::new(pos_args, None, kw_args, Some(paren)); function.call_expr(args) } - ExpressionType::Binop { a, op, b } => { + py_ast::Expr::Binop { a, op, b } => { let lhs = self.convert_expr(*a); let rhs = self.convert_expr(*b); let op = op_to_token(op); Expr::BinOp(BinOp::new(op, lhs, rhs)) } - ExpressionType::Unop { op, a } => { + py_ast::Expr::Unop { op, a } => { let rhs = self.convert_expr(*a); let (kind, cont) = match op { - UnaryOperator::Pos => (TokenKind::PrePlus, "+"), - // UnaryOperator::Not => (TokenKind::PreBitNot, "not"), - UnaryOperator::Neg => (TokenKind::PreMinus, "-"), - UnaryOperator::Inv => (TokenKind::PreBitNot, "~"), + Unaryop::Pos => (TokenKind::PrePlus, "+"), + // Unaryop::Not => (TokenKind::PreBitNot, "not"), + Unaryop::Neg => (TokenKind::PreMinus, "-"), + Unaryop::Inv => (TokenKind::PreBitNot, "~"), _ => return Expr::Dummy(Dummy::new(None, vec![rhs])), }; let op = Token::from_str(kind, cont); Expr::UnaryOp(UnaryOp::new(op, rhs)) } // TODO - ExpressionType::BoolOp { op, mut values } => { + py_ast::Expr::BoolOp { op, mut values } => { let lhs = self.convert_expr(values.remove(0)); let rhs = self.convert_expr(values.remove(0)); let (kind, cont) = match op { - BooleanOperator::And => (TokenKind::AndOp, "and"), - BooleanOperator::Or => (TokenKind::OrOp, "or"), + Boolop::And => (TokenKind::AndOp, "and"), + Boolop::Or => (TokenKind::OrOp, "or"), }; let op = Token::from_str(kind, cont); Expr::BinOp(BinOp::new(op, lhs, rhs)) } - // TODO: multiple comparisons - ExpressionType::Compare { mut vals, mut ops } => { + // TODO: multiple Cmpops + py_ast::Expr::Compare { mut vals, mut ops } => { let lhs = self.convert_expr(vals.remove(0)); let rhs = self.convert_expr(vals.remove(0)); let (kind, cont) = match ops.remove(0) { - Comparison::Equal => (TokenKind::DblEq, "=="), - Comparison::NotEqual => (TokenKind::NotEq, "!="), - Comparison::Less => (TokenKind::Less, "<"), - Comparison::LessOrEqual => (TokenKind::LessEq, "<="), - Comparison::Greater => (TokenKind::Gre, ">"), - Comparison::GreaterOrEqual => (TokenKind::GreEq, ">="), - Comparison::Is => (TokenKind::IsOp, "is!"), - Comparison::IsNot => (TokenKind::IsNotOp, "isnot!"), - Comparison::In => (TokenKind::InOp, "in"), - Comparison::NotIn => (TokenKind::NotInOp, "notin"), + Cmpop::Eq => (TokenKind::DblEq, "=="), + Cmpop::NotEq => (TokenKind::NotEq, "!="), + Cmpop::Lt => (TokenKind::Less, "<"), + Cmpop::LtE => (TokenKind::LessEq, "<="), + Cmpop::Gt => (TokenKind::Gre, ">"), + Cmpop::GtE => (TokenKind::GreEq, ">="), + Cmpop::Is => (TokenKind::IsOp, "is!"), + Cmpop::IsNot => (TokenKind::IsNotOp, "isnot!"), + Cmpop::In => (TokenKind::InOp, "in"), + Cmpop::NotIn => (TokenKind::NotInOp, "notin"), }; let op = Token::from_str(kind, cont); Expr::BinOp(BinOp::new(op, lhs, rhs)) } - ExpressionType::Lambda { args, body } => { + py_ast::Expr::Lambda { args, body } => { self.namespace.push("".to_string()); let params = self.convert_params(*args); let body = vec![self.convert_expr(*body)]; @@ -1063,9 +1060,12 @@ impl ASTConverter { let op = Token::from_str(TokenKind::FuncArrow, "->"); Expr::Lambda(Lambda::new(sig, op, Block::new(body), DefId(0))) } - ExpressionType::List { elements } => { - let (l_sqbr, r_sqbr) = - Self::gen_enclosure_tokens(TokenKind::LSqBr, elements.iter(), expr.location); + py_ast::Expr::List { elements } => { + let (l_sqbr, r_sqbr) = Self::gen_enclosure_tokens( + TokenKind::LSqBr, + elements.iter(), + expr.location().start(), + ); let elements = elements .into_iter() .map(|ex| PosArg::new(self.convert_expr(ex))) @@ -1074,9 +1074,12 @@ impl ASTConverter { let arr = Expr::Array(Array::Normal(NormalArray::new(l_sqbr, r_sqbr, elems))); Self::mutate_expr(arr) } - ExpressionType::Set { elements } => { - let (l_brace, r_brace) = - Self::gen_enclosure_tokens(TokenKind::LBrace, elements.iter(), expr.location); + py_ast::Expr::Set { elements } => { + let (l_brace, r_brace) = Self::gen_enclosure_tokens( + TokenKind::LBrace, + elements.iter(), + expr.location().start(), + ); let elements = elements .into_iter() .map(|ex| PosArg::new(self.convert_expr(ex))) @@ -1085,11 +1088,11 @@ impl ASTConverter { Expr::Set(Set::Normal(NormalSet::new(l_brace, r_brace, elems))) // Self::mutate_expr(set) } - ExpressionType::Dict { elements } => { + py_ast::Expr::Dict { elements } => { let (l_brace, r_brace) = Self::gen_enclosure_tokens( TokenKind::LBrace, elements.iter().map(|(_, v)| v), - expr.location, + expr.location().start(), ); let kvs = elements .into_iter() @@ -1104,7 +1107,7 @@ impl ASTConverter { let dict = Expr::Dict(Dict::Normal(NormalDict::new(l_brace, r_brace, kvs))); Self::mutate_expr(dict) } - ExpressionType::Tuple { elements } => { + py_ast::Expr::Tuple { elements } => { let elements = elements .into_iter() .map(|ex| PosArg::new(self.convert_expr(ex))) @@ -1112,10 +1115,11 @@ impl ASTConverter { let elems = Args::pos_only(elements, None); Expr::Tuple(Tuple::Normal(NormalTuple::new(elems))) } - ExpressionType::Subscript { a, b } => { + py_ast::Expr::Subscript { a, b } => { let obj = self.convert_expr(*a); - let method = - obj.attr_expr(self.convert_ident("__getitem__".to_string(), expr.location)); + let method = obj.attr_expr( + self.convert_ident("__getitem__".to_string(), expr.location().start()), + ); method.call1(self.convert_expr(*b)) } _other => { @@ -1125,7 +1129,7 @@ impl ASTConverter { } } - fn convert_block(&mut self, block: Vec>, kind: BlockKind) -> Block { + fn convert_block(&mut self, block: Suite, kind: BlockKind) -> Block { let mut new_block = Vec::new(); let len = block.len(); self.block_id_counter += 1; @@ -1304,7 +1308,7 @@ impl ASTConverter { fn extract_method( &mut self, - body: Vec>, + body: Vec, inherit: bool, ) -> (Option, ClassAttrs) { let mut base_type = None; @@ -1386,7 +1390,7 @@ impl ASTConverter { fn extract_method_list( &mut self, ident: Identifier, - body: Vec>, + body: Vec, inherit: bool, ) -> (Option, Vec) { let class = TypeSpec::mono(ident.clone()); @@ -1400,10 +1404,10 @@ impl ASTConverter { &mut self, name: String, params: Parameters, - body: Vec>, - decorator_list: Vec>, - returns: Option>, - loc: PyLocation, + body: Vec, + decorator_list: Vec, + returns: Option, + range: PySourceRange, ) -> Expr { // if reassigning of a function referenced by other functions is occurred, it is an error if self @@ -1417,13 +1421,14 @@ impl ASTConverter { { let err = reassign_func_error( self.cfg.input.clone(), - pyloc_to_ergloc(loc, name.len()), + pyloc_to_ergloc(range), self.namespace.join("."), &name, ); self.errs.push(err); Expr::Dummy(Dummy::new(None, vec![])) } else { + let loc = range.start; let decos = decorator_list .into_iter() .map(|ex| Decorator(self.convert_expr(ex))) @@ -1466,9 +1471,9 @@ impl ASTConverter { fn convert_classdef( &mut self, name: String, - body: Vec>, - bases: Vec>, - decorator_list: Vec>, + body: Vec, + bases: Vec, + decorator_list: Vec, loc: PyLocation, ) -> Expr { let _decos = decorator_list @@ -1523,19 +1528,19 @@ impl ASTConverter { Expr::ClassDef(classdef) } - fn convert_statement(&mut self, stmt: Located, dont_call_return: bool) -> Expr { + fn convert_statement(&mut self, stmt: Stmt, dont_call_return: bool) -> Expr { match stmt.node { - StatementType::Expression { expression } => self.convert_expr(expression), - StatementType::AnnAssign { + py_ast::Stmt::Expression(stmt) => self.convert_expr(stmt.expr), + py_ast::Stmt::AnnAssign { target, annotation, value, } => { - let anot = self.convert_expr(clone_loc_expr(&annotation)); + let anot = self.convert_expr(annotation.clone()); let t_spec = self.convert_type_spec(*annotation); let t_spec = TypeSpecWithOp::new(AS, t_spec, anot); match target.node { - ExpressionType::Identifier { name } => { + py_ast::Expr::Identifier { name } => { if let Some(value) = value { let block = Block::new(vec![self.convert_expr(value)]); let body = DefBody::new(EQUAL, block, DefId(0)); @@ -1556,7 +1561,7 @@ impl ASTConverter { Expr::TypeAscription(tasc) } } - ExpressionType::Attribute { value: attr, name } => { + py_ast::Expr::Attribute { value: attr, name } => { let attr = self .convert_expr(*attr) .attr(self.convert_attr_ident(name, target.location)); @@ -1572,11 +1577,11 @@ impl ASTConverter { _other => Expr::Dummy(Dummy::new(None, vec![])), } } - StatementType::Assign { mut targets, value } => { + py_ast::Stmt::Assign { mut targets, value } => { if targets.len() == 1 { let lhs = targets.remove(0); match lhs.node { - ExpressionType::Identifier { name } => { + py_ast::Expr::Identifier { name } => { let block = Block::new(vec![self.convert_expr(value)]); let body = DefBody::new(EQUAL, block, DefId(0)); self.register_name_info(&name, NameKind::Variable); @@ -1586,11 +1591,12 @@ impl ASTConverter { let def = Def::new(sig, body); Expr::Def(def) } - ExpressionType::Attribute { value: attr, name } => { - let attr_name_loc = PyLocation::new( - attr.location.row(), - attr.location.column() + length(&attr.node) + 1, - ); + py_ast::Expr::Attribute(py_ast::ExprAttribute { + value: attr, + name, + range, + }) => { + let attr_name_loc = attr.end_location(); let attr = self .convert_expr(*attr) .attr(self.convert_attr_ident(name, attr_name_loc)); @@ -1598,7 +1604,7 @@ impl ASTConverter { let adef = ReDef::new(attr, expr); Expr::ReDef(adef) } - ExpressionType::Tuple { elements } => { + py_ast::Expr::Tuple { elements } => { let tmp = fresh_varname(); let tmp_name = VarName::from_str_and_line( (&tmp).into(), @@ -1644,7 +1650,7 @@ impl ASTConverter { } // a[b] = x // => a.__setitem__(b, x) - ExpressionType::Subscript { a, b } => { + py_ast::Expr::Subscript { a, b } => { let a = self.convert_expr(*a); let b = self.convert_expr(*b); let x = self.convert_expr(value); @@ -1663,7 +1669,7 @@ impl ASTConverter { let mut defs = vec![]; for target in targets { match target.node { - ExpressionType::Identifier { name } => { + py_ast::Expr::Identifier { name } => { let body = DefBody::new(EQUAL, Block::new(vec![value.clone()]), DefId(0)); self.register_name_info(&name, NameKind::Variable); @@ -1683,10 +1689,10 @@ impl ASTConverter { Expr::Dummy(Dummy::new(None, defs)) } } - StatementType::AugAssign { target, op, value } => { + py_ast::Stmt::AugAssign { target, op, value } => { let op = op_to_token(op); match target.node { - ExpressionType::Identifier { name } => { + py_ast::Expr::Identifier { name } => { let val = self.convert_expr(*value); let prev_ident = self.convert_ident(name.clone(), stmt.location); if self @@ -1712,7 +1718,7 @@ impl ASTConverter { Expr::ReDef(redef) } } - ExpressionType::Attribute { value: attr, name } => { + py_ast::Expr::Attribute { value: attr, name } => { let val = self.convert_expr(*value); let attr = self .convert_expr(*attr) @@ -1727,22 +1733,24 @@ impl ASTConverter { } } } - StatementType::FunctionDef { + py_ast::Stmt::FunctionDef { is_async: _, name, args, body, decorator_list, returns, - } => self.convert_funcdef(name, *args, body, decorator_list, returns, stmt.location), - StatementType::ClassDef { + range, + } => self.convert_funcdef(name, *args, body, decorator_list, returns, range), + py_ast::Stmt::ClassDef { name, body, bases, keywords: _, decorator_list, - } => self.convert_classdef(name, body, bases, decorator_list, stmt.location), - StatementType::For { + range, + } => self.convert_classdef(name, body, bases, decorator_list, range), + py_ast::Stmt::For { is_async: _, target, iter, @@ -1755,7 +1763,7 @@ impl ASTConverter { let for_acc = Expr::Accessor(Accessor::Ident(for_ident)); for_acc.call2(iter, Expr::Lambda(block)) } - StatementType::While { + py_ast::Stmt::While { test, body, orelse: _, @@ -1769,7 +1777,7 @@ impl ASTConverter { let while_acc = Expr::Accessor(Accessor::Ident(while_ident)); while_acc.call2(test, Expr::Lambda(body)) } - StatementType::If { test, body, orelse } => { + py_ast::Stmt::If { test, body, orelse } => { let block = self.convert_block(body, BlockKind::If); let params = Params::new(vec![], None, vec![], None); let sig = LambdaSignature::new(params.clone(), None, TypeBoundSpecs::empty()); @@ -1794,7 +1802,7 @@ impl ASTConverter { if_acc.call2(test, Expr::Lambda(body)) } } - StatementType::Return { value } => { + py_ast::Stmt::Return { value } => { let value = value .map(|val| self.convert_expr(val)) .unwrap_or_else(|| Expr::Tuple(Tuple::Normal(NormalTuple::new(Args::empty())))); @@ -1809,7 +1817,7 @@ impl ASTConverter { return_acc.call1(value) } } - StatementType::Assert { test, msg } => { + py_ast::Stmt::Assert { test, msg } => { let test = self.convert_expr(test); let args = if let Some(msg) = msg { let msg = self.convert_expr(msg); @@ -1822,7 +1830,7 @@ impl ASTConverter { )); assert_acc.call_expr(args) } - StatementType::Import { names } => { + py_ast::Stmt::Import { names } => { let mut imports = vec![]; for name in names { let import_acc = Expr::Accessor(Accessor::Ident( @@ -1862,12 +1870,12 @@ impl ASTConverter { Expr::Dummy(Dummy::new(None, imports)) } // from module import foo, bar - StatementType::ImportFrom { + py_ast::Stmt::ImportFrom { level: _, module, names, } => self.convert_from_import(module, names, stmt.location), - StatementType::Try { + py_ast::Stmt::Try { body, handlers: _, orelse, @@ -1889,7 +1897,7 @@ impl ASTConverter { }; Expr::Dummy(Dummy::new(None, dummy)) } - StatementType::With { + py_ast::Stmt::With { is_async: _, mut items, body, @@ -2017,9 +2025,9 @@ impl ASTConverter { } } - pub fn convert_program(mut self, program: Program) -> IncompleteArtifact { + pub fn convert_program(mut self, program: ModModule) -> IncompleteArtifact { let program = program - .statements + .body .into_iter() .map(|stmt| self.convert_statement(stmt, true)) .collect(); diff --git a/crates/py2erg/lib.rs b/crates/py2erg/lib.rs index 00abe1c..c0381f6 100644 --- a/crates/py2erg/lib.rs +++ b/crates/py2erg/lib.rs @@ -1,5 +1,4 @@ pub mod ast_util; -mod clone; mod convert; mod error; mod gen_decl; diff --git a/src/analyze.rs b/src/analyze.rs index 5f21739..b8086ff 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -7,14 +7,14 @@ use erg_common::Str; use erg_compiler::artifact::{BuildRunnable, Buildable, CompleteArtifact, IncompleteArtifact}; use erg_compiler::context::register::CheckStatus; use erg_compiler::context::ModuleContext; -use erg_compiler::erg_parser::ast::{AST, Module}; +use erg_compiler::erg_parser::ast::{Module, AST}; use erg_compiler::erg_parser::error::ParseErrors; use erg_compiler::erg_parser::parse::Parsable; use erg_compiler::error::{CompileError, CompileErrors}; use erg_compiler::lower::ASTLowerer; use erg_compiler::module::SharedCompilerResource; use py2erg::{dump_decl_er, reserve_decl_er, ShadowingMode}; -use rustpython_parser::parser; +use rustpython_parser::{Parse, ast as py_ast}; use crate::handle_err; @@ -22,9 +22,8 @@ pub struct SimplePythonParser {} impl Parsable for SimplePythonParser { fn parse(code: String) -> Result { - let py_program = parser::parse_program(&code).map_err(|_err| { - ParseErrors::empty() - })?; + let py_program = py_ast::Suite::parse(&code).map_err(|_err| ParseErrors::empty())?; + // TODO: SourceLocator let shadowing = if cfg!(feature = "debug") { ShadowingMode::Visible } else { From e550dd986e4ced21182f7d3aa39a96e50d2f1498 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Mon, 29 May 2023 06:30:41 +0900 Subject: [PATCH 2/8] Update convert.rs --- crates/py2erg/convert.rs | 487 ++++++++++++++++++--------------------- 1 file changed, 220 insertions(+), 267 deletions(-) diff --git a/crates/py2erg/convert.rs b/crates/py2erg/convert.rs index ec2d589..98de5da 100644 --- a/crates/py2erg/convert.rs +++ b/crates/py2erg/convert.rs @@ -23,8 +23,8 @@ use erg_compiler::erg_parser::token::{Token, TokenKind, AS, DOT, EQUAL}; use erg_compiler::erg_parser::Parser; use erg_compiler::error::{CompileError, CompileErrors}; use rustpython_parser::ast::located::{ - self as py_ast, Boolop, Cmpop, ImportSymbol, Keyword, Located, ModModule, Number, Operator, - Parameter, Parameters, StringGroup, Suite, Suite, Unaryop, + self as py_ast, Boolop, Cmpop, StmtImport, Keyword, Located, ModModule, Operator, + String, Suite, Unaryop, Arguments, Arg, Stmt, ExprConstant }; use rustpython_parser::source_code::{ SourceLocation as PyLocation, SourceLocator, SourceRange as PySourceRange, @@ -120,10 +120,10 @@ fn op_to_token(op: Operator) -> Token { pub fn pyloc_to_ergloc(range: PySourceRange) -> erg_common::error::Location { erg_common::error::Location::range( - loc.start.row() as u32, - loc.start.column() as u32, - loc.end.row() as u32, - loc.end.column() as u32, + range.start.row.get(), + range.start.column.get(), + range.end.unwrap().row.get(), + range.end.unwrap().column.get(), ) } @@ -327,15 +327,15 @@ impl ASTConverter { let token = Token::new( TokenKind::Symbol, cont, - loc.row() as u32, - loc.column() as u32 - 1, + loc.row.get(), + loc.column.get() - 1, ); let name = VarName::new(token); let dot = Token::new( TokenKind::Dot, ".", - loc.row() as u32, - loc.column() as u32 - 1, + loc.row.get(), + loc.column.get() - 1, ); Identifier::new(VisModifierSpec::Public(dot), name) } @@ -345,15 +345,15 @@ impl ASTConverter { let token = Token::new( TokenKind::Symbol, name, - loc.row() as u32, - loc.column() as u32 - 1, + loc.row.get() as u32, + loc.column.get() as u32 - 1, ); let name = VarName::new(token); let dot = Token::new( TokenKind::Dot, ".", - loc.row() as u32, - loc.column() as u32 - 1, + loc.row.get() as u32, + loc.column.get() as u32 - 1, ); Identifier::new(VisModifierSpec::Public(dot), name) } @@ -364,13 +364,13 @@ impl ASTConverter { ParamPattern::VarName(ident.name) } - fn convert_nd_param(&mut self, param: Parameter) -> NonDefaultParamSignature { - let pat = self.convert_param_pattern(param.arg, param.location); + fn convert_nd_param(&mut self, param: Arg) -> NonDefaultParamSignature { + let pat = self.convert_param_pattern(param.arg.to_string(), param.location()); let t_spec = param .annotation .map(|anot| { ( - self.convert_type_spec(anot.clone()), + self.convert_type_spec(*anot.clone()), self.convert_expr(*anot), ) }) @@ -380,7 +380,7 @@ impl ASTConverter { fn convert_default_param( &mut self, - kw: Parameter, + kw: Arg, default: py_ast::Expr, ) -> DefaultParamSignature { let sig = self.convert_nd_param(kw); @@ -388,7 +388,7 @@ impl ASTConverter { DefaultParamSignature::new(sig, default) } - fn convert_params(&mut self, params: Parameters) -> Params { + fn convert_params(&mut self, params: Arguments) -> Params { let non_defaults_len = params.args.len() - params.defaults.len(); let mut non_default_names = params.args; let defaults_names = non_default_names.split_off(non_defaults_len); @@ -442,13 +442,13 @@ impl ASTConverter { ) -> (NonDefaultParamSignature, Vec) { match expr { py_ast::Expr::Name(expr) => ( - self.convert_for_param(expr.name, expr.location().start()), + self.convert_for_param(expr.name, expr.location()), vec![], ), py_ast::Expr::Tuple(expr) => { let tmp = fresh_varname(); let tmp_name = - VarName::from_str_and_line((&tmp).into(), expr.location().start().row() as u32); + VarName::from_str_and_line((&tmp).into(), expr.location().row.get()); let tmp_expr = Expr::Accessor(Accessor::Ident(Identifier::new( VisModifierSpec::Public(DOT), tmp_name.clone(), @@ -458,8 +458,8 @@ impl ASTConverter { let index = Literal::new(Token::new( TokenKind::NatLit, i.to_string(), - elem.location.row() as u32, - elem.location.column() as u32 - 1, + elem.location().row.get(), + elem.location().column.get() as u32 - 1, )); let (param, mut blocks) = self.convert_expr_to_param(elem); let sig = Signature::Var(VarSignature::new( @@ -467,7 +467,7 @@ impl ASTConverter { param.t_spec, )); let method = tmp_expr.clone().attr_expr( - self.convert_ident("__getitem__".to_string(), expr.location().start()), + self.convert_ident("__getitem__".to_string(), expr.location()), ); let tuple_acc = method.call1(Expr::Literal(index)); let body = DefBody::new(EQUAL, Block::new(vec![tuple_acc]), DefId(0)); @@ -482,8 +482,8 @@ impl ASTConverter { let token = Token::new( TokenKind::UBar, "_", - expr.location().start().row() as u32, - expr.location().start().column() as u32 - 1, + expr.location().row.get(), + expr.location().column.get() - 1, ); ( NonDefaultParamSignature::new(ParamPattern::Discard(token), None), @@ -517,8 +517,8 @@ impl ASTConverter { TypeSpec::Infer(Token::new( TokenKind::UBar, "_", - loc.row() as u32, - loc.column() as u32 - 1, + loc.row.get(), + loc.column.get() - 1, )) } @@ -526,7 +526,7 @@ impl ASTConverter { fn convert_compound_type_spec(&mut self, name: String, args: py_ast::Expr) -> TypeSpec { match &name[..] { "Union" => { - let py_ast::Expr::Tuple { mut elements } = args.node else { + let py_ast::Expr::Tuple(tuple) = args else { let err = CompileError::syntax_error( self.cfg.input.clone(), line!() as usize, @@ -536,30 +536,30 @@ impl ASTConverter { None, ); self.errs.push(err); - return Self::gen_dummy_type_spec(args.location); + return Self::gen_dummy_type_spec(args.location()); }; - let lhs = self.convert_type_spec(elements.remove(0)); - let rhs = self.convert_type_spec(elements.remove(0)); + let lhs = self.convert_type_spec(tuple.elts.remove(0)); + let rhs = self.convert_type_spec(tuple.elts.remove(0)); let mut union = TypeSpec::or(lhs, rhs); - for elem in elements { + for elem in tuple.elts { let t = self.convert_type_spec(elem); union = TypeSpec::or(union, t); } union } "Optional" => { - let loc = args.location; + let loc = args.location(); let t = self.convert_type_spec(args); let ident = Identifier::private_with_line("NoneType".into(), loc.row() as u32); let none = TypeSpec::mono(ident); TypeSpec::or(t, none) } "Literal" => { - let py_ast::Expr::Tuple { elements } = args.node else { - return Self::gen_dummy_type_spec(args.location); + let py_ast::Expr::Tuple(tuple) = args else { + return Self::gen_dummy_type_spec(args.location()); }; let mut elems = vec![]; - for elem in elements { + for elem in tuple.elts { let expr = self.convert_expr(elem); match Parser::validate_const_expr(expr) { Ok(expr) => { @@ -580,14 +580,14 @@ impl ASTConverter { } // TODO: distinguish from collections.abc.Callable "Callable" => { - let py_ast::Expr::Tuple { mut elements } = args.node else { - return Self::gen_dummy_type_spec(args.location); + let py_ast::Expr::Tuple(tuple) = args else { + return Self::gen_dummy_type_spec(args.location()); }; - let params = elements.remove(0); + let params = tuple.elts.remove(0); let mut non_defaults = vec![]; - match params.node { - py_ast::Expr::List { elements } => { - for param in elements.into_iter() { + match params { + py_ast::Expr::List(list) => { + for param in list.elts.into_iter() { let t_spec = self.convert_type_spec(param); non_defaults.push(ParamTySpec::anonymous(t_spec)); } @@ -604,7 +604,7 @@ impl ASTConverter { self.errs.push(err); } } - let ret = self.convert_type_spec(elements.remove(0)); + let ret = self.convert_type_spec(tuple.elts.remove(0)); TypeSpec::Subr(SubrTypeSpec::new( TypeBoundSpecs::empty(), None, @@ -640,19 +640,19 @@ impl ASTConverter { TypeSpec::poly(acc, ConstArgs::pos_only(vec![elem_t], None)) } "Mapping" | "MutableMapping" => { - let py_ast::Expr::Tuple { mut elements } = args.node else { + let py_ast::Expr::Tuple(tuple) = args else { let err = CompileError::syntax_error( self.cfg.input.clone(), line!() as usize, - pyloc_to_ergloc(args.location.range()), + pyloc_to_ergloc(args.range()), self.cur_namespace(), format!("`{name}` takes 2 types"), None, ); self.errs.push(err); - return Self::gen_dummy_type_spec(args.location); + return Self::gen_dummy_type_spec(args.location()); }; - let key_t = self.convert_expr(elements.remove(0)); + let key_t = self.convert_expr(tuple.elts.remove(0)); let key_t = match Parser::validate_const_expr(key_t) { Ok(key_t) => key_t, Err(err) => { @@ -666,7 +666,7 @@ impl ASTConverter { } }; let key_t = ConstPosArg::new(key_t); - let value_t = self.convert_expr(elements.remove(0)); + let value_t = self.convert_expr(tuple.elts.remove(0)); let value_t = match Parser::validate_const_expr(value_t) { Ok(value_t) => value_t, Err(err) => { @@ -690,7 +690,7 @@ impl ASTConverter { } "list" => { let len = ConstExpr::Accessor(ConstAccessor::Local( - self.convert_ident("_".into(), args.location), + self.convert_ident("_".into(), args.location()), )); let elem_t = self.convert_expr(args); let elem_t = match Parser::validate_const_expr(elem_t) { @@ -716,12 +716,12 @@ impl ASTConverter { TypeSpec::poly(acc, ConstArgs::new(vec![elem_t, len], None, vec![], None)) } "dict" => { - let py_ast::Expr::Tuple { mut elements } = args.node else { - return Self::gen_dummy_type_spec(args.location); + let py_ast::Expr::Tuple(tuple) = args else { + return Self::gen_dummy_type_spec(args.location()); }; let (l_brace, r_brace) = - Self::gen_enclosure_tokens(TokenKind::LBrace, elements.iter(), args.location); - let key_t = self.convert_expr(elements.remove(0)); + Self::gen_enclosure_tokens(TokenKind::LBrace, tuple.elts.iter(), args.location()); + let key_t = self.convert_expr(tuple.elts.remove(0)); let key_t = match Parser::validate_const_expr(key_t) { Ok(key_t) => key_t, Err(err) => { @@ -734,7 +734,7 @@ impl ASTConverter { ConstExpr::Accessor(ConstAccessor::Local(Identifier::private("Obj".into()))) } }; - let val_t = self.convert_expr(elements.remove(0)); + let val_t = self.convert_expr(tuple.elts.remove(0)); let val_t = match Parser::validate_const_expr(val_t) { Ok(val_t) => val_t, Err(err) => { @@ -761,40 +761,40 @@ impl ASTConverter { TypeSpec::poly(acc, ConstArgs::new(vec![dict], None, vec![], None)) } "tuple" => { - let py_ast::Expr::Tuple { elements } = args.node else { - return Self::gen_dummy_type_spec(args.location); + let py_ast::Expr::Tuple(tuple) = args else { + return Self::gen_dummy_type_spec(args.location()); }; let parens = - Self::gen_enclosure_tokens(TokenKind::LParen, elements.iter(), args.location); - let tys = elements + Self::gen_enclosure_tokens(TokenKind::LParen, tuple.elts.iter(), args.location()); + let tys = tuple.elts .into_iter() .map(|elem| self.convert_type_spec(elem)) .collect(); let tuple = TupleTypeSpec::new(Some(parens), tys); TypeSpec::Tuple(tuple) } - _ => Self::gen_dummy_type_spec(args.location), + _ => Self::gen_dummy_type_spec(args.location()), } } fn convert_type_spec(&mut self, expr: py_ast::Expr) -> TypeSpec { #[allow(clippy::collapsible_match)] - match expr.node { - py_ast::Expr::Identifier { name } => { - self.convert_ident_type_spec(name, expr.location().start()) + match expr { + py_ast::Expr::Name(name) => { + self.convert_ident_type_spec(name.id.to_string(), expr.location()) } - py_ast::Expr::None => { - self.convert_ident_type_spec("NoneType".into(), expr.location().start()) + py_ast::Expr::Constant(cons) => { + self.convert_ident_type_spec("NoneType".into(), expr.location()) } - py_ast::Expr::Attribute { value, name } => { - let namespace = Box::new(self.convert_expr(*value)); - let t = self.convert_ident(name, expr.location().start()); + py_ast::Expr::Attribute(attr) => { + let namespace = Box::new(self.convert_expr(*attr.value)); + let t = self.convert_ident(attr.attr, expr.location()); let predecl = PreDeclTypeSpec::Attr { namespace, t }; TypeSpec::PreDeclTy(predecl) } - py_ast::Expr::Subscript { a, b } => match a.node { - py_ast::Expr::Identifier { name } => self.convert_compound_type_spec(name, *b), - py_ast::Expr::Attribute { value, name } => { + py_ast::Expr::Subscript(subs) => match *subs.value { + py_ast::Expr::Name(name) => self.convert_compound_type_spec(name, *subs.slice), + py_ast::Expr::Attribute(attr) => { match accessor_name(value.node).as_ref().map(|s| &s[..]) { Some("typing" | "collections.abc") => { self.convert_compound_type_spec(name, *b) @@ -810,20 +810,20 @@ impl ASTConverter { Self::gen_dummy_type_spec(a.location) } }, - py_ast::Expr::Binop { a, op, b } => { - match op { + py_ast::Expr::BinOp(bin) => { + match bin.op { // A | B Operator::BitOr => { let lhs = self.convert_type_spec(*a); let rhs = self.convert_type_spec(*b); TypeSpec::or(lhs, rhs) } - _ => Self::gen_dummy_type_spec(expr.location().start()), + _ => Self::gen_dummy_type_spec(expr.location()), } } other => { log!(err "unknown: {other:?}"); - Self::gen_dummy_type_spec(expr.location().start()) + Self::gen_dummy_type_spec(expr.location()) } } } @@ -843,16 +843,16 @@ impl ASTConverter { _ => unreachable!(), }; let (l_end, c_end) = if elems.len() == 0 { - (expr_loc.row(), expr_loc.column() - 1) + (expr_loc.row.get(), expr_loc.column.get() - 1) } else { let last = elems.last().unwrap(); - (last.location.row(), last.location.column()) + (last.location().row.get(), last.location().column.get()) }; let l_brace = Token::new( l_kind, l_cont, - expr_loc.row() as u32, - expr_loc.column() as u32 - 1, + expr_loc.row.get(), + expr_loc.column.get() - 1, ); let r_brace = Token::new(r_kind, r_cont, l_end as u32, c_end as u32); (l_brace, r_brace) @@ -869,7 +869,7 @@ impl ASTConverter { } fn convert_expr(&mut self, expr: py_ast::Expr) -> Expr { - match expr.node { + match expr { py_ast::Expr::Number { value } => { let (kind, cont) = match value { Number::Integer { value } if value >= 0.into() => { @@ -889,6 +889,8 @@ impl ASTConverter { ); Expr::Literal(Literal::new(token)) } + py_ast::Expr::Constant(cons) => match cons.value { + } py_ast::Expr::String { value } => { let StringGroup::Constant{ value } = value else { return Expr::Dummy(Dummy::new(None, vec![])); @@ -927,11 +929,11 @@ impl ASTConverter { expr.location().start().row() as u32, expr.location().start().column() as u32 - 1, ))), - py_ast::Expr::Identifier { name } => { + py_ast::Expr::Name(name) => { let ident = self.convert_ident(name, expr.location().start()); Expr::Accessor(Accessor::Ident(ident)) } - py_ast::Expr::Attribute { value, name } => { + py_ast::Expr::Attribute(attr) => { let obj = self.convert_expr(*value); let attr_name_loc = PyLocation::new( obj.ln_end().unwrap_or(1) as usize, @@ -940,15 +942,15 @@ impl ASTConverter { let name = self.convert_attr_ident(name, attr_name_loc); obj.attr_expr(name) } - py_ast::Expr::IfExpression { test, body, orelse } => { - let block = self.convert_expr(*body); + py_ast::Expr::IfExp(if_) => { + let block = self.convert_expr(*if_.body); let params = Params::new(vec![], None, vec![], None); let sig = LambdaSignature::new(params.clone(), None, TypeBoundSpecs::empty()); let body = Lambda::new(sig, Token::DUMMY, Block::new(vec![block]), DefId(0)); - let test = self.convert_expr(*test); - let if_ident = self.convert_ident("if".to_string(), expr.location().start()); + let test = self.convert_expr(*if_.test); + let if_ident = self.convert_ident("if".to_string(), expr.location()); let if_acc = Expr::Accessor(Accessor::Ident(if_ident)); - let else_block = self.convert_expr(*orelse); + let else_block = self.convert_expr(*if_.orelse); let sig = LambdaSignature::new(params, None, TypeBoundSpecs::empty()); let else_body = Lambda::new(sig, Token::DUMMY, Block::new(vec![else_block]), DefId(0)); @@ -962,17 +964,13 @@ impl ASTConverter { ); if_acc.call_expr(args) } - py_ast::Expr::Call { - function, - args, - keywords, - } => { - let function = self.convert_expr(*function); - let pos_args = args + py_ast::Expr::Call(call) => { + let function = self.convert_expr(*call.func); + let pos_args = call.args .into_iter() .map(|ex| PosArg::new(self.convert_expr(ex))) .collect::>(); - let kw_args = keywords + let kw_args = call.keywords .into_iter() .map(|Keyword { arg, value, range }| { let name = arg.unwrap_or_default(); @@ -989,13 +987,13 @@ impl ASTConverter { let lp = Token::new( TokenKind::LParen, "(", - expr.location().start().row() as u32, + expr.location().row.get(), function.col_end().unwrap_or(0) + 1, ); let rp = Token::new( TokenKind::RParen, ")", - expr.location().start().row() as u32, + expr.location().row.get(), last_col + 1, ); (lp, rp) @@ -1003,29 +1001,29 @@ impl ASTConverter { let args = Args::new(pos_args, None, kw_args, Some(paren)); function.call_expr(args) } - py_ast::Expr::Binop { a, op, b } => { + py_ast::Expr::BinOp(bin) => { let lhs = self.convert_expr(*a); let rhs = self.convert_expr(*b); let op = op_to_token(op); Expr::BinOp(BinOp::new(op, lhs, rhs)) } - py_ast::Expr::Unop { op, a } => { + py_ast::Expr::UnaryOp(un) => { let rhs = self.convert_expr(*a); - let (kind, cont) = match op { - Unaryop::Pos => (TokenKind::PrePlus, "+"), + let (kind, cont) = match un.op { + Unaryop::UAdd => (TokenKind::PrePlus, "+"), // Unaryop::Not => (TokenKind::PreBitNot, "not"), - Unaryop::Neg => (TokenKind::PreMinus, "-"), - Unaryop::Inv => (TokenKind::PreBitNot, "~"), + Unaryop::USub => (TokenKind::PreMinus, "-"), + Unaryop::Invert => (TokenKind::PreBitNot, "~"), _ => return Expr::Dummy(Dummy::new(None, vec![rhs])), }; let op = Token::from_str(kind, cont); Expr::UnaryOp(UnaryOp::new(op, rhs)) } // TODO - py_ast::Expr::BoolOp { op, mut values } => { - let lhs = self.convert_expr(values.remove(0)); - let rhs = self.convert_expr(values.remove(0)); - let (kind, cont) = match op { + py_ast::Expr::BoolOp(bool) => { + let lhs = self.convert_expr(bool.values.remove(0)); + let rhs = self.convert_expr(bool.values.remove(0)); + let (kind, cont) = match bool.op { Boolop::And => (TokenKind::AndOp, "and"), Boolop::Or => (TokenKind::OrOp, "or"), }; @@ -1033,10 +1031,10 @@ impl ASTConverter { Expr::BinOp(BinOp::new(op, lhs, rhs)) } // TODO: multiple Cmpops - py_ast::Expr::Compare { mut vals, mut ops } => { - let lhs = self.convert_expr(vals.remove(0)); - let rhs = self.convert_expr(vals.remove(0)); - let (kind, cont) = match ops.remove(0) { + py_ast::Expr::Compare(cmp) => { + let lhs = self.convert_expr(*cmp.left); + let rhs = self.convert_expr(cmp.comparators.remove(0)); + let (kind, cont) = match cmp.ops.remove(0) { Cmpop::Eq => (TokenKind::DblEq, "=="), Cmpop::NotEq => (TokenKind::NotEq, "!="), Cmpop::Lt => (TokenKind::Less, "<"), @@ -1051,22 +1049,22 @@ impl ASTConverter { let op = Token::from_str(kind, cont); Expr::BinOp(BinOp::new(op, lhs, rhs)) } - py_ast::Expr::Lambda { args, body } => { + py_ast::Expr::Lambda(lambda) => { self.namespace.push("".to_string()); - let params = self.convert_params(*args); - let body = vec![self.convert_expr(*body)]; + let params = self.convert_params(*lambda.args); + let body = vec![self.convert_expr(*lambda.body)]; self.namespace.pop(); let sig = LambdaSignature::new(params, None, TypeBoundSpecs::empty()); let op = Token::from_str(TokenKind::FuncArrow, "->"); Expr::Lambda(Lambda::new(sig, op, Block::new(body), DefId(0))) } - py_ast::Expr::List { elements } => { + py_ast::Expr::List(list) => { let (l_sqbr, r_sqbr) = Self::gen_enclosure_tokens( TokenKind::LSqBr, - elements.iter(), - expr.location().start(), + list.elts.iter(), + expr.location(), ); - let elements = elements + let elements = list.elts .into_iter() .map(|ex| PosArg::new(self.convert_expr(ex))) .collect::>(); @@ -1074,13 +1072,13 @@ impl ASTConverter { let arr = Expr::Array(Array::Normal(NormalArray::new(l_sqbr, r_sqbr, elems))); Self::mutate_expr(arr) } - py_ast::Expr::Set { elements } => { + py_ast::Expr::Set(set) => { let (l_brace, r_brace) = Self::gen_enclosure_tokens( TokenKind::LBrace, - elements.iter(), - expr.location().start(), + set.elts.iter(), + expr.location(), ); - let elements = elements + let elements = set.elts .into_iter() .map(|ex| PosArg::new(self.convert_expr(ex))) .collect::>(); @@ -1088,14 +1086,15 @@ impl ASTConverter { Expr::Set(Set::Normal(NormalSet::new(l_brace, r_brace, elems))) // Self::mutate_expr(set) } - py_ast::Expr::Dict { elements } => { + py_ast::Expr::Dict(dict) => { let (l_brace, r_brace) = Self::gen_enclosure_tokens( TokenKind::LBrace, - elements.iter().map(|(_, v)| v), - expr.location().start(), + dict.values.iter(), + expr.location(), ); - let kvs = elements + let kvs = dict.keys .into_iter() + .zip(dict.values.into_iter()) .map(|(k, v)| { KeyValue::new( k.map(|k| self.convert_expr(k)) @@ -1107,18 +1106,18 @@ impl ASTConverter { let dict = Expr::Dict(Dict::Normal(NormalDict::new(l_brace, r_brace, kvs))); Self::mutate_expr(dict) } - py_ast::Expr::Tuple { elements } => { - let elements = elements + py_ast::Expr::Tuple(tuple) => { + let elements = tuple.elts .into_iter() .map(|ex| PosArg::new(self.convert_expr(ex))) .collect::>(); let elems = Args::pos_only(elements, None); Expr::Tuple(Tuple::Normal(NormalTuple::new(elems))) } - py_ast::Expr::Subscript { a, b } => { + py_ast::Expr::Subscript(subs) => { let obj = self.convert_expr(*a); let method = obj.attr_expr( - self.convert_ident("__getitem__".to_string(), expr.location().start()), + self.convert_ident("__getitem__".to_string(), expr.location()), ); method.call1(self.convert_expr(*b)) } @@ -1403,7 +1402,7 @@ impl ASTConverter { fn convert_funcdef( &mut self, name: String, - params: Parameters, + params: Arguments, body: Vec, decorator_list: Vec, returns: Option, @@ -1434,7 +1433,7 @@ impl ASTConverter { .map(|ex| Decorator(self.convert_expr(ex))) .collect::>(); self.register_name_info(&name, NameKind::Function); - let func_name_loc = PyLocation::new(loc.row(), loc.column() + 4); + let func_name_loc = PyLocation{ row: loc.row.get(), column: loc.column.get() + 4 }; let ident = self.convert_ident(name, func_name_loc); self.namespace.push(ident.inspect().to_string()); let params = self.convert_params(params); @@ -1529,24 +1528,20 @@ impl ASTConverter { } fn convert_statement(&mut self, stmt: Stmt, dont_call_return: bool) -> Expr { - match stmt.node { - py_ast::Stmt::Expression(stmt) => self.convert_expr(stmt.expr), - py_ast::Stmt::AnnAssign { - target, - annotation, - value, - } => { - let anot = self.convert_expr(annotation.clone()); - let t_spec = self.convert_type_spec(*annotation); + match stmt { + py_ast::Stmt::Expr(stmt) => self.convert_expr(*stmt.value), + py_ast::Stmt::AnnAssign(ann_assign) => { + let anot = self.convert_expr(*ann_assign.annotation.clone()); + let t_spec = self.convert_type_spec(*ann_assign.annotation); let t_spec = TypeSpecWithOp::new(AS, t_spec, anot); - match target.node { - py_ast::Expr::Identifier { name } => { - if let Some(value) = value { - let block = Block::new(vec![self.convert_expr(value)]); + match *ann_assign.target { + py_ast::Expr::Name(name) => { + if let Some(value) = ann_assign.value { + let block = Block::new(vec![self.convert_expr(*value)]); let body = DefBody::new(EQUAL, block, DefId(0)); // must register after convert_expr because value may be contain name (e.g. i = i + 1) - self.register_name_info(&name, NameKind::Variable); - let ident = self.convert_ident(name, stmt.location); + self.register_name_info(name.id.as_str(), NameKind::Variable); + let ident = self.convert_ident(name.id.to_string(), stmt.location()); let sig = Signature::Var(VarSignature::new( VarPattern::Ident(ident), Some(t_spec), @@ -1555,18 +1550,18 @@ impl ASTConverter { Expr::Def(def) } else { // no registration because it's just a type ascription - let ident = self.convert_ident(name, stmt.location); + let ident = self.convert_ident(name, stmt.location()); let tasc = TypeAscription::new(Expr::Accessor(Accessor::Ident(ident)), t_spec); Expr::TypeAscription(tasc) } } - py_ast::Expr::Attribute { value: attr, name } => { + py_ast::Expr::Attribute(attr) => { let attr = self - .convert_expr(*attr) - .attr(self.convert_attr_ident(name, target.location)); - if let Some(value) = value { - let expr = self.convert_expr(value); + .convert_expr(*attr.value) + .attr(self.convert_attr_ident(attr.attr, ann_assign.target.location())); + if let Some(value) = ann_assign.value { + let expr = self.convert_expr(*value); let redef = ReDef::new(attr, expr); Expr::ReDef(redef) } else { @@ -1577,12 +1572,12 @@ impl ASTConverter { _other => Expr::Dummy(Dummy::new(None, vec![])), } } - py_ast::Stmt::Assign { mut targets, value } => { - if targets.len() == 1 { - let lhs = targets.remove(0); - match lhs.node { - py_ast::Expr::Identifier { name } => { - let block = Block::new(vec![self.convert_expr(value)]); + py_ast::Stmt::Assign(assign) => { + if assign.targets.len() == 1 { + let lhs = assign.targets.remove(0); + match lhs { + py_ast::Expr::Name(name) => { + let block = Block::new(vec![self.convert_expr(*assign.value)]); let body = DefBody::new(EQUAL, block, DefId(0)); self.register_name_info(&name, NameKind::Variable); let ident = self.convert_ident(name, stmt.location); @@ -1591,24 +1586,20 @@ impl ASTConverter { let def = Def::new(sig, body); Expr::Def(def) } - py_ast::Expr::Attribute(py_ast::ExprAttribute { - value: attr, - name, - range, - }) => { + py_ast::Expr::Attribute(attr) => { let attr_name_loc = attr.end_location(); let attr = self - .convert_expr(*attr) + .convert_expr(*attr.value) .attr(self.convert_attr_ident(name, attr_name_loc)); - let expr = self.convert_expr(value); + let expr = self.convert_expr(*assign.value); let adef = ReDef::new(attr, expr); Expr::ReDef(adef) } - py_ast::Expr::Tuple { elements } => { + py_ast::Expr::Tuple(tuple) => { let tmp = fresh_varname(); let tmp_name = VarName::from_str_and_line( (&tmp).into(), - stmt.location.row() as u32, + stmt.location().row.get(), ); let tmp_ident = Identifier::new(VisModifierSpec::Public(DOT), tmp_name); let tmp_expr = Expr::Accessor(Accessor::Ident(tmp_ident.clone())); @@ -1618,17 +1609,17 @@ impl ASTConverter { )); let body = DefBody::new( EQUAL, - Block::new(vec![self.convert_expr(value)]), + Block::new(vec![self.convert_expr(*assign.value)]), DefId(0), ); let tmp_def = Expr::Def(Def::new(sig, body)); let mut defs = vec![tmp_def]; - for (i, elem) in elements.into_iter().enumerate() { + for (i, elem) in tuple.elts.into_iter().enumerate() { let index = Literal::new(Token::new( TokenKind::NatLit, i.to_string(), - elem.location.row() as u32, - elem.location.column() as u32 - 1, + elem.location().row.get(), + elem.location().column.get() - 1, )); let (param, mut blocks) = self.convert_opt_expr_to_param(Some(elem)); @@ -1637,7 +1628,7 @@ impl ASTConverter { param.t_spec, )); let method = tmp_expr.clone().attr_expr( - self.convert_ident("__getitem__".to_string(), stmt.location), + self.convert_ident("__getitem__".to_string(), stmt.location()), ); let tuple_acc = method.call1(Expr::Literal(index)); let body = @@ -1650,12 +1641,12 @@ impl ASTConverter { } // a[b] = x // => a.__setitem__(b, x) - py_ast::Expr::Subscript { a, b } => { - let a = self.convert_expr(*a); - let b = self.convert_expr(*b); - let x = self.convert_expr(value); + py_ast::Expr::Subscript(subs) => { + let a = self.convert_expr(*subs.value); + let b = self.convert_expr(*subs.slice); + let x = self.convert_expr(*assign.value); let method = a.attr_expr( - self.convert_ident("__setitem__".to_string(), stmt.location), + self.convert_ident("__setitem__".to_string(), stmt.location()), ); method.call2(b, x) } @@ -1665,15 +1656,15 @@ impl ASTConverter { } } } else { - let value = self.convert_expr(value); + let value = self.convert_expr(*assign.value); let mut defs = vec![]; - for target in targets { - match target.node { - py_ast::Expr::Identifier { name } => { + for target in assign.targets { + match target { + py_ast::Expr::Name(name) => { let body = DefBody::new(EQUAL, Block::new(vec![value.clone()]), DefId(0)); self.register_name_info(&name, NameKind::Variable); - let ident = self.convert_ident(name, stmt.location); + let ident = self.convert_ident(name, stmt.location()); let sig = Signature::Var(VarSignature::new( VarPattern::Ident(ident), None, @@ -1689,19 +1680,19 @@ impl ASTConverter { Expr::Dummy(Dummy::new(None, defs)) } } - py_ast::Stmt::AugAssign { target, op, value } => { - let op = op_to_token(op); - match target.node { + py_ast::Stmt::AugAssign(aug_assign) => { + let op = op_to_token(aug_assign.op); + match aug_assign.target { py_ast::Expr::Identifier { name } => { let val = self.convert_expr(*value); - let prev_ident = self.convert_ident(name.clone(), stmt.location); + let prev_ident = self.convert_ident(name.clone(), stmt.location()); if self .get_name(&name) .map(|info| info.defined_block_id == self.cur_block_id()) .unwrap_or(false) { self.register_name_info(&name, NameKind::Variable); - let ident = self.convert_ident(name.clone(), stmt.location); + let ident = self.convert_ident(name.clone(), stmt.location()); let bin = BinOp::new(op, Expr::Accessor(Accessor::Ident(prev_ident)), val); let sig = @@ -1711,14 +1702,14 @@ impl ASTConverter { let def = Def::new(sig, body); Expr::Def(def) } else { - let ident = self.convert_ident(name.clone(), stmt.location); + let ident = self.convert_ident(name.clone(), stmt.location()); let bin = BinOp::new(op, Expr::Accessor(Accessor::Ident(prev_ident)), val); let redef = ReDef::new(Accessor::Ident(ident), Expr::BinOp(bin)); Expr::ReDef(redef) } } - py_ast::Expr::Attribute { value: attr, name } => { + py_ast::Expr::Attribute(attr) => { let val = self.convert_expr(*value); let attr = self .convert_expr(*attr) @@ -1733,59 +1724,34 @@ impl ASTConverter { } } } - py_ast::Stmt::FunctionDef { - is_async: _, - name, - args, - body, - decorator_list, - returns, - range, - } => self.convert_funcdef(name, *args, body, decorator_list, returns, range), - py_ast::Stmt::ClassDef { - name, - body, - bases, - keywords: _, - decorator_list, - range, - } => self.convert_classdef(name, body, bases, decorator_list, range), - py_ast::Stmt::For { - is_async: _, - target, - iter, - body, - orelse: _, - } => { - let iter = self.convert_expr(*iter); - let block = self.convert_for_body(Some(*target), body); - let for_ident = self.convert_ident("for".to_string(), stmt.location); + py_ast::Stmt::FunctionDef(func_def) => self.convert_funcdef(name, *args, body, decorator_list, returns, range), + py_ast::Stmt::ClassDef(class_def) => self.convert_classdef(name, body, bases, decorator_list, range), + py_ast::Stmt::For(for_) => { + let iter = self.convert_expr(*for_.iter); + let block = self.convert_for_body(Some(*for_.target), for_.body); + let for_ident = self.convert_ident("for".to_string(), stmt.location()); let for_acc = Expr::Accessor(Accessor::Ident(for_ident)); for_acc.call2(iter, Expr::Lambda(block)) } - py_ast::Stmt::While { - test, - body, - orelse: _, - } => { - let test = self.convert_expr(test); + py_ast::Stmt::While(while_) => { + let test = self.convert_expr(*while_.test); let params = Params::new(vec![], None, vec![], None); let empty_sig = LambdaSignature::new(params, None, TypeBoundSpecs::empty()); - let block = self.convert_block(body, BlockKind::While); + let block = self.convert_block(while_.body, BlockKind::While); let body = Lambda::new(empty_sig, Token::DUMMY, block, DefId(0)); - let while_ident = self.convert_ident("while".to_string(), stmt.location); + let while_ident = self.convert_ident("while".to_string(), stmt.location()); let while_acc = Expr::Accessor(Accessor::Ident(while_ident)); while_acc.call2(test, Expr::Lambda(body)) } - py_ast::Stmt::If { test, body, orelse } => { - let block = self.convert_block(body, BlockKind::If); + py_ast::Stmt::If(if_) => { + let block = self.convert_block(if_.body, BlockKind::If); let params = Params::new(vec![], None, vec![], None); let sig = LambdaSignature::new(params.clone(), None, TypeBoundSpecs::empty()); let body = Lambda::new(sig, Token::DUMMY, block, DefId(0)); - let test = self.convert_expr(test); - let if_ident = self.convert_ident("if".to_string(), stmt.location); + let test = self.convert_expr(*if_.test); + let if_ident = self.convert_ident("if".to_string(), stmt.location()); let if_acc = Expr::Accessor(Accessor::Ident(if_ident)); - if let Some(orelse) = orelse { + if let Some(orelse) = if_.orelse { let else_block = self.convert_block(orelse, BlockKind::If); let sig = LambdaSignature::new(params, None, TypeBoundSpecs::empty()); let else_body = Lambda::new(sig, Token::DUMMY, else_block, DefId(0)); @@ -1802,52 +1768,52 @@ impl ASTConverter { if_acc.call2(test, Expr::Lambda(body)) } } - py_ast::Stmt::Return { value } => { - let value = value + py_ast::Stmt::Return(return_) => { + let value = return_.value .map(|val| self.convert_expr(val)) .unwrap_or_else(|| Expr::Tuple(Tuple::Normal(NormalTuple::new(Args::empty())))); if dont_call_return { value } else { let func_acc = Expr::Accessor(Accessor::Ident( - self.convert_ident(self.namespace.last().unwrap().clone(), stmt.location), + self.convert_ident(self.namespace.last().unwrap().clone(), stmt.location()), )); - let return_acc = self.convert_ident("return".to_string(), stmt.location); + let return_acc = self.convert_ident("return".to_string(), stmt.location()); let return_acc = Expr::Accessor(Accessor::attr(func_acc, return_acc)); return_acc.call1(value) } } - py_ast::Stmt::Assert { test, msg } => { - let test = self.convert_expr(test); - let args = if let Some(msg) = msg { - let msg = self.convert_expr(msg); + py_ast::Stmt::Assert(assert) => { + let test = self.convert_expr(*assert.test); + let args = if let Some(msg) = assert.msg { + let msg = self.convert_expr(*msg); Args::pos_only(vec![PosArg::new(test), PosArg::new(msg)], None) } else { Args::pos_only(vec![PosArg::new(test)], None) }; let assert_acc = Expr::Accessor(Accessor::Ident( - self.convert_ident("assert".to_string(), stmt.location), + self.convert_ident("assert".to_string(), stmt.location()), )); assert_acc.call_expr(args) } - py_ast::Stmt::Import { names } => { + py_ast::Stmt::Import(import) => { let mut imports = vec![]; - for name in names { + for name in import.names { let import_acc = Expr::Accessor(Accessor::Ident( - self.convert_ident("__import__".to_string(), stmt.location), + self.convert_ident("__import__".to_string(), stmt.location()), )); - let cont = format!("\"{}\"", name.symbol.split('.').next().unwrap()); + let cont = format!("\"{}\"", name.name.split('.').next().unwrap()); let mod_name = Expr::Literal(Literal::new(Token::new( TokenKind::StrLit, cont, - stmt.location.row() as u32, - stmt.location.column() as u32 - 1, + stmt.location().row.get(), + stmt.location().column.get() - 1, ))); let call = import_acc.call1(mod_name); - let def = if let Some(alias) = name.alias { + let def = if let Some(alias) = name.asname { self.register_name_info(&alias, NameKind::Variable); let var = VarSignature::new( - VarPattern::Ident(self.convert_ident(alias, stmt.location)), + VarPattern::Ident(self.convert_ident(alias, stmt.location())), None, ); Def::new( @@ -1855,9 +1821,9 @@ impl ASTConverter { DefBody::new(EQUAL, Block::new(vec![call]), DefId(0)), ) } else { - self.register_name_info(&name.symbol, NameKind::Variable); + self.register_name_info(&name.name, NameKind::Variable); let var = VarSignature::new( - VarPattern::Ident(self.convert_ident(name.symbol, stmt.location)), + VarPattern::Ident(self.convert_ident(name.name, stmt.location())), None, ); Def::new( @@ -1870,17 +1836,8 @@ impl ASTConverter { Expr::Dummy(Dummy::new(None, imports)) } // from module import foo, bar - py_ast::Stmt::ImportFrom { - level: _, - module, - names, - } => self.convert_from_import(module, names, stmt.location), - py_ast::Stmt::Try { - body, - handlers: _, - orelse, - finalbody, - } => { + py_ast::Stmt::ImportFrom(import_from) => self.convert_from_import(module, names, stmt.location), + py_ast::Stmt::Try(try_) => { let chunks = self.convert_block(body, BlockKind::Try).into_iter(); let dummy = match (orelse, finalbody) { (Some(orelse), Some(finalbody)) => chunks @@ -1897,15 +1854,11 @@ impl ASTConverter { }; Expr::Dummy(Dummy::new(None, dummy)) } - py_ast::Stmt::With { - is_async: _, - mut items, - body, - } => { - let item = items.remove(0); + py_ast::Stmt::With(with) => { + let item = with.items.remove(0); let context_expr = self.convert_expr(item.context_expr); let body = self.convert_for_body(item.optional_vars, body); - let with_ident = self.convert_ident("with".to_string(), stmt.location); + let with_ident = self.convert_ident("with".to_string(), stmt.location()); let with_acc = Expr::Accessor(Accessor::Ident(with_ident)); with_acc.call2(context_expr, Expr::Lambda(body)) } From d4c4f08a6f1bb2e90c65986d898c398349b69074 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Wed, 7 Jun 2023 02:54:00 +0900 Subject: [PATCH 3/8] trial --- Cargo.lock | 401 +++++++++++++++++++++++++++----------- Cargo.toml | 10 +- crates/py2erg/Cargo.toml | 1 - crates/py2erg/ast_util.rs | 19 +- crates/py2erg/convert.rs | 370 ++++++++++++++++++++--------------- src/analyze.rs | 2 +- 6 files changed, 504 insertions(+), 299 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 064389d..cb84057 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,12 +61,34 @@ dependencies = [ "nix", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" + [[package]] name = "cc" version = "1.0.79" @@ -79,12 +101,49 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "cpufeatures" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +dependencies = [ + "libc", +] + [[package]] name = "crunchy" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "either" version = "1.8.1" @@ -93,9 +152,9 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "els" -version = "0.1.26-nightly.0" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf50f787a8376384af8e7b074c489329669eeb1264f15f1a7f136102a003f2f" +checksum = "ad69bdb783d8f54317278b3d32b138b05fe0708c9a22454cd9505f39edf4d044" dependencies = [ "erg_common", "erg_compiler", @@ -105,22 +164,31 @@ dependencies = [ ] [[package]] -name = "erg_common" -version = "0.6.14-nightly.0" +name = "embed-doc-image" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0d10554accd6de5e3f41e3db5931348320c89b9745a82635887f850dd77aadb" +checksum = "af36f591236d9d822425cb6896595658fa558fcebf5ee8accac1d4b92c47166e" +dependencies = [ + "base64", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "erg_common" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abef65c68abb0bbf94d43bdedddc44349ff02f8a21b510dedf66f4be24587a0c" dependencies = [ "backtrace-on-stack-overflow", - "hermit-abi", - "libc", - "winapi", ] [[package]] name = "erg_compiler" -version = "0.6.14-nightly.0" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3570aa70c9f488516c6286280ae4d3343aed382b6c7881b784195af64350aa50" +checksum = "09c82d01efbb07c5da0d5ee35ce6fc053e09512dca0275136e5cf2c07bd7ee43" dependencies = [ "erg_common", "erg_parser", @@ -128,9 +196,9 @@ dependencies = [ [[package]] name = "erg_parser" -version = "0.6.14-nightly.0" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd44c03719e8051c7997263298e2b792da6138f818646d0eeae359f92bf0c60" +checksum = "b572d85a2a33d729a8dd03a655ee1c36b9e86f4e85deb22f9043542e4b8fb41f" dependencies = [ "erg_common", "unicode-xid", @@ -145,27 +213,33 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "gimli" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hexf-parse" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" - [[package]] name = "idna" version = "0.3.0" @@ -189,6 +263,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.10.5" @@ -204,42 +287,21 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +[[package]] +name = "keccak" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +dependencies = [ + "cpufeatures", +] + [[package]] name = "lalrpop-util" version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d" -[[package]] -name = "lexical-parse-float" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" -dependencies = [ - "lexical-parse-integer", - "lexical-util", - "static_assertions", -] - -[[package]] -name = "lexical-parse-integer" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" -dependencies = [ - "lexical-util", - "static_assertions", -] - -[[package]] -name = "lexical-util" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" -dependencies = [ - "static_assertions", -] - [[package]] name = "libc" version = "0.2.144" @@ -269,10 +331,63 @@ dependencies = [ ] [[package]] -name = "matches" -version = "0.1.10" +name = "malachite" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +checksum = "f6cf7f4730c30071ba374fac86ad35b1cb7a0716f774737768667ea3fa1828e3" +dependencies = [ + "malachite-base", + "malachite-nz", + "malachite-q", +] + +[[package]] +name = "malachite-base" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b06bfa98a4b4802af5a4263b4ad4660e28e51e8490f6354eb9336c70767e1c5" +dependencies = [ + "itertools 0.9.0", + "rand 0.7.3", + "rand_chacha", + "ryu", + "sha3", +] + +[[package]] +name = "malachite-bigint" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a5110aee54537b0cef214efbebdd7df79b7408db8eef4f6a4b6db9d0d8fc01b" +dependencies = [ + "derive_more", + "malachite", + "num-integer", + "num-traits", + "paste", +] + +[[package]] +name = "malachite-nz" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e21c64b7af5be3dc8cef16f786243faf59459fe4ba93b44efdeb264e5ade4" +dependencies = [ + "embed-doc-image", + "itertools 0.9.0", + "malachite-base", +] + +[[package]] +name = "malachite-q" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3755e541d5134b5016594c9043094172c4dda9259b3ce824a7b8101941850360" +dependencies = [ + "itertools 0.9.0", + "malachite-base", + "malachite-nz", +] [[package]] name = "memchr" @@ -311,17 +426,6 @@ dependencies = [ "memoffset", ] -[[package]] -name = "num-bigint" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - [[package]] name = "num-integer" version = "0.1.45" @@ -356,6 +460,18 @@ version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "paste" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" + [[package]] name = "percent-encoding" version = "2.2.0" @@ -388,7 +504,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] @@ -411,6 +527,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "proc-macro2" version = "1.0.58" @@ -426,7 +548,6 @@ version = "0.0.29" dependencies = [ "erg_common", "erg_compiler", - "rustpython-ast", "rustpython-parser", ] @@ -450,13 +571,45 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", ] [[package]] @@ -465,9 +618,19 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "ruff_source_location" version = "0.0.0" +source = "git+https://github.com/RustPython/Parser#b2f95e284852cb25d6e510c127260d03573d9f0c" dependencies = [ "memchr", "once_cell", @@ -477,6 +640,7 @@ dependencies = [ [[package]] name = "ruff_text_size" version = "0.0.0" +source = "git+https://github.com/RustPython/Parser#b2f95e284852cb25d6e510c127260d03573d9f0c" [[package]] name = "rustc-demangle" @@ -490,38 +654,37 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustpython-ast" version = "0.2.0" +source = "git+https://github.com/RustPython/Parser#b2f95e284852cb25d6e510c127260d03573d9f0c" dependencies = [ "is-macro", - "num-bigint", - "rustpython-literal", + "malachite-bigint", "rustpython-parser-core", "static_assertions", ] -[[package]] -name = "rustpython-literal" -version = "0.2.0" -dependencies = [ - "hexf-parse", - "is-macro", - "lexical-parse-float", - "num-traits", - "unic-ucd-category", -] - [[package]] name = "rustpython-parser" version = "0.2.0" +source = "git+https://github.com/RustPython/Parser#b2f95e284852cb25d6e510c127260d03573d9f0c" dependencies = [ "anyhow", "is-macro", - "itertools", + "itertools 0.10.5", "lalrpop-util", "log", - "num-bigint", + "malachite-bigint", "num-traits", "phf", "phf_codegen", @@ -537,8 +700,10 @@ dependencies = [ [[package]] name = "rustpython-parser-core" version = "0.2.0" +source = "git+https://github.com/RustPython/Parser#b2f95e284852cb25d6e510c127260d03573d9f0c" dependencies = [ "is-macro", + "memchr", "ruff_source_location", "ruff_text_size", ] @@ -549,6 +714,12 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "semver" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" + [[package]] name = "serde" version = "1.0.163" @@ -591,6 +762,18 @@ dependencies = [ "syn 2.0.16", ] +[[package]] +name = "sha3" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809" +dependencies = [ + "block-buffer", + "digest", + "keccak", + "opaque-debug", +] + [[package]] name = "siphasher" version = "0.3.10" @@ -649,6 +832,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + [[package]] name = "unic-char-property" version = "0.9.0" @@ -681,18 +870,6 @@ dependencies = [ "unic-ucd-version", ] -[[package]] -name = "unic-ucd-category" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8d4591f5fcfe1bd4453baaf803c40e1b1e69ff8455c47620440b46efef91c0" -dependencies = [ - "matches", - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-ident" version = "0.9.0" @@ -761,23 +938,13 @@ dependencies = [ ] [[package]] -name = "winapi" -version = "0.3.9" +name = "version_check" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" diff --git a/Cargo.toml b/Cargo.toml index c228898..8b7ed51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,12 +22,10 @@ edition = "2021" repository = "https://github.com/mtshiba/pylyzer" [workspace.dependencies] -erg_common = { version = "0.6.14-nightly.0", features = ["py_compat", "els"] } -erg_compiler = { version = "0.6.14-nightly.0", features = ["py_compat", "els"] } -els = { version = "0.1.26-nightly.0", features = ["py_compat"] } -# rustpython-parser = { git = "https://github.com/RustPython/Parser.git", rev = "4de0cb18277f45aecd1cfdcc5ac69ab6bd9e2854", features = ["rustpython-literal"] } -rustpython-ast = { path = "../RustPython-Parser/ast", features = ["rustpython-literal"] } -rustpython-parser = { path = "../RustPython-Parser/parser" } +erg_common = { version = "0.6.14", features = ["py_compat", "els"] } +erg_compiler = { version = "0.6.14", features = ["py_compat", "els"] } +els = { version = "0.1.26", features = ["py_compat"] } +rustpython-parser = { git = "https://github.com/RustPython/Parser", version = "0.2.0", features = ["all-nodes-with-ranges"] } # erg_compiler = { git = "https://github.com/erg-lang/erg", branch = "main", features = ["py_compat", "els"] } # erg_common = { git = "https://github.com/erg-lang/erg", branch = "main", features = ["py_compat", "els"] } # els = { git = "https://github.com/erg-lang/erg", branch = "main", features = ["py_compat"] } diff --git a/crates/py2erg/Cargo.toml b/crates/py2erg/Cargo.toml index 94461c3..77471b6 100644 --- a/crates/py2erg/Cargo.toml +++ b/crates/py2erg/Cargo.toml @@ -14,7 +14,6 @@ debug = ["erg_compiler/debug", "erg_common/debug"] [dependencies] rustpython-parser = { workspace = true } -rustpython-ast = { workspace = true } erg_common = { workspace = true } erg_compiler = { workspace = true } diff --git a/crates/py2erg/ast_util.rs b/crates/py2erg/ast_util.rs index d784758..5dec411 100644 --- a/crates/py2erg/ast_util.rs +++ b/crates/py2erg/ast_util.rs @@ -1,19 +1,10 @@ -use erg_common::log; -use rustpython_parser::ast::{Boolop, Cmpop, Constant, ExpressionType, Keyword, StringGroup}; +use rustpython_parser::ast::Expr; -pub fn number_to_string(num: &Constant) -> String { - num.to_string() -} - -pub fn comp_to_string(comp: &Cmpop) -> String { - cmpop.as_str() -} - -pub fn accessor_name(expr: ExpressionType) -> Option { +pub fn accessor_name(expr: Expr) -> Option { match expr { - ExpressionType::Identifier { name } => Some(name), - ExpressionType::Attribute { value, name } => { - accessor_name(value.node).map(|value| format!("{value}.{name}")) + Expr::Name(name) => Some(name.id.to_string()), + Expr::Attribute(attr) => { + accessor_name(*attr.value).map(|value| format!("{value}.{}", attr.attr)) } _ => None, } diff --git a/crates/py2erg/convert.rs b/crates/py2erg/convert.rs index 98de5da..895b079 100644 --- a/crates/py2erg/convert.rs +++ b/crates/py2erg/convert.rs @@ -19,16 +19,14 @@ use erg_compiler::erg_parser::ast::{ VisModifierSpec, }; use erg_compiler::erg_parser::desugar::Desugarer; -use erg_compiler::erg_parser::token::{Token, TokenKind, AS, DOT, EQUAL}; +use erg_compiler::erg_parser::token::{Token, TokenKind, AS, COLON, DOT, EQUAL}; use erg_compiler::erg_parser::Parser; use erg_compiler::error::{CompileError, CompileErrors}; use rustpython_parser::ast::located::{ - self as py_ast, Boolop, Cmpop, StmtImport, Keyword, Located, ModModule, Operator, - String, Suite, Unaryop, Arguments, Arg, Stmt, ExprConstant -}; -use rustpython_parser::source_code::{ - SourceLocation as PyLocation, SourceLocator, SourceRange as PySourceRange, + self as py_ast, Alias, Arg, Arguments, Boolop, Cmpop, ExprConstant, Keyword, Located, + ModModule, Operator, Stmt, String, Suite, Unaryop, }; +use rustpython_parser::source_code::{SourceLocation as PyLocation, SourceRange as PySourceRange}; use crate::ast_util::accessor_name; use crate::error::*; @@ -328,14 +326,14 @@ impl ASTConverter { TokenKind::Symbol, cont, loc.row.get(), - loc.column.get() - 1, + loc.column.to_zero_indexed(), ); let name = VarName::new(token); let dot = Token::new( TokenKind::Dot, ".", loc.row.get(), - loc.column.get() - 1, + loc.column.to_zero_indexed(), ); Identifier::new(VisModifierSpec::Public(dot), name) } @@ -345,15 +343,15 @@ impl ASTConverter { let token = Token::new( TokenKind::Symbol, name, - loc.row.get() as u32, - loc.column.get() as u32 - 1, + loc.row.get(), + loc.column.to_zero_indexed(), ); let name = VarName::new(token); let dot = Token::new( TokenKind::Dot, ".", - loc.row.get() as u32, - loc.column.get() as u32 - 1, + loc.row.get(), + loc.column.to_zero_indexed(), ); Identifier::new(VisModifierSpec::Public(dot), name) } @@ -378,28 +376,25 @@ impl ASTConverter { NonDefaultParamSignature::new(pat, t_spec) } - fn convert_default_param( - &mut self, - kw: Arg, - default: py_ast::Expr, - ) -> DefaultParamSignature { + fn convert_default_param(&mut self, kw: Arg, default: py_ast::Expr) -> DefaultParamSignature { let sig = self.convert_nd_param(kw); let default = self.convert_expr(default); DefaultParamSignature::new(sig, default) } fn convert_params(&mut self, params: Arguments) -> Params { - let non_defaults_len = params.args.len() - params.defaults.len(); - let mut non_default_names = params.args; - let defaults_names = non_default_names.split_off(non_defaults_len); - let non_defaults = non_default_names + let non_defaults_len = params.args.len() - params.defaults(); + let mut args = params.into_python_arguments(); + let defaults = params.defaults(); + let non_defaults = args + .args .into_iter() .map(|p| self.convert_nd_param(p)) .collect(); let defaults = defaults_names .into_iter() - .zip(params.defaults.into_iter()) - .map(|(kw, default)| self.convert_default_param(kw, default)) + .zip(params.defaults()) + .map(|(kw, default)| self.convert_default_param(kw, default.clone())) .collect(); Params::new(non_defaults, None, defaults, None) } @@ -442,13 +437,12 @@ impl ASTConverter { ) -> (NonDefaultParamSignature, Vec) { match expr { py_ast::Expr::Name(expr) => ( - self.convert_for_param(expr.name, expr.location()), + self.convert_for_param(expr.id.to_string(), expr.location()), vec![], ), py_ast::Expr::Tuple(expr) => { let tmp = fresh_varname(); - let tmp_name = - VarName::from_str_and_line((&tmp).into(), expr.location().row.get()); + let tmp_name = VarName::from_str_and_line((&tmp).into(), expr.location().row.get()); let tmp_expr = Expr::Accessor(Accessor::Ident(Identifier::new( VisModifierSpec::Public(DOT), tmp_name.clone(), @@ -466,9 +460,9 @@ impl ASTConverter { Self::param_pattern_to_var(param.pat), param.t_spec, )); - let method = tmp_expr.clone().attr_expr( - self.convert_ident("__getitem__".to_string(), expr.location()), - ); + let method = tmp_expr + .clone() + .attr_expr(self.convert_ident("__getitem__".to_string(), expr.location())); let tuple_acc = method.call1(Expr::Literal(index)); let body = DefBody::new(EQUAL, Block::new(vec![tuple_acc]), DefId(0)); let def = Expr::Def(Def::new(sig, body)); @@ -550,7 +544,7 @@ impl ASTConverter { "Optional" => { let loc = args.location(); let t = self.convert_type_spec(args); - let ident = Identifier::private_with_line("NoneType".into(), loc.row() as u32); + let ident = Identifier::private_with_line("NoneType".into(), loc.row.get()); let none = TypeSpec::mono(ident); TypeSpec::or(t, none) } @@ -719,8 +713,11 @@ impl ASTConverter { let py_ast::Expr::Tuple(tuple) = args else { return Self::gen_dummy_type_spec(args.location()); }; - let (l_brace, r_brace) = - Self::gen_enclosure_tokens(TokenKind::LBrace, tuple.elts.iter(), args.location()); + let (l_brace, r_brace) = Self::gen_enclosure_tokens( + TokenKind::LBrace, + tuple.elts.iter(), + args.location(), + ); let key_t = self.convert_expr(tuple.elts.remove(0)); let key_t = match Parser::validate_const_expr(key_t) { Ok(key_t) => key_t, @@ -764,9 +761,13 @@ impl ASTConverter { let py_ast::Expr::Tuple(tuple) = args else { return Self::gen_dummy_type_spec(args.location()); }; - let parens = - Self::gen_enclosure_tokens(TokenKind::LParen, tuple.elts.iter(), args.location()); - let tys = tuple.elts + let parens = Self::gen_enclosure_tokens( + TokenKind::LParen, + tuple.elts.iter(), + args.location(), + ); + let tys = tuple + .elts .into_iter() .map(|elem| self.convert_type_spec(elem)) .collect(); @@ -788,34 +789,37 @@ impl ASTConverter { } py_ast::Expr::Attribute(attr) => { let namespace = Box::new(self.convert_expr(*attr.value)); - let t = self.convert_ident(attr.attr, expr.location()); + let t = self.convert_ident(attr.attr.to_string(), expr.location()); let predecl = PreDeclTypeSpec::Attr { namespace, t }; TypeSpec::PreDeclTy(predecl) } + // value[slice] py_ast::Expr::Subscript(subs) => match *subs.value { - py_ast::Expr::Name(name) => self.convert_compound_type_spec(name, *subs.slice), + py_ast::Expr::Name(name) => { + self.convert_compound_type_spec(name.id.to_string(), *subs.slice) + } py_ast::Expr::Attribute(attr) => { - match accessor_name(value.node).as_ref().map(|s| &s[..]) { + match accessor_name(attr.value).as_ref().map(|s| &s[..]) { Some("typing" | "collections.abc") => { - self.convert_compound_type_spec(name, *b) + self.convert_compound_type_spec(attr.attr.to_string(), *subs.slice) } _ => { - log!(err "unknown: .{name}"); - Self::gen_dummy_type_spec(a.location) + log!(err "unknown: {subs:?}"); + Self::gen_dummy_type_spec(subs.value.location()) } } } other => { log!(err "unknown: {other:?}"); - Self::gen_dummy_type_spec(a.location) + Self::gen_dummy_type_spec(other.location()) } }, py_ast::Expr::BinOp(bin) => { match bin.op { // A | B Operator::BitOr => { - let lhs = self.convert_type_spec(*a); - let rhs = self.convert_type_spec(*b); + let lhs = self.convert_type_spec(*bin.left); + let rhs = self.convert_type_spec(*bin.right); TypeSpec::or(lhs, rhs) } _ => Self::gen_dummy_type_spec(expr.location()), @@ -868,78 +872,86 @@ impl ASTConverter { Expr::UnaryOp(UnaryOp::new(mut_op, expr)) } - fn convert_expr(&mut self, expr: py_ast::Expr) -> Expr { - match expr { - py_ast::Expr::Number { value } => { - let (kind, cont) = match value { - Number::Integer { value } if value >= 0.into() => { - (TokenKind::NatLit, value.to_string()) - } - Number::Integer { value } => (TokenKind::IntLit, value.to_string()), - Number::Float { value } => (TokenKind::RatioLit, value.to_string()), - Number::Complex { .. } => { - return Expr::Dummy(Dummy::new(None, vec![])); - } + fn convert_const(&mut self, const_: ExprConstant) -> Expr { + match const_.value { + py_ast::Constant::Int(i) => { + let kind = if i >= 0.into() { + TokenKind::NatLit + } else { + TokenKind::IntLit }; let token = Token::new( kind, - cont, - expr.location().start().row() as u32, - expr.location().start().column() as u32 - 1, + i.to_string(), + const_.location().row.get(), + const_.location().column.to_zero_indexed(), ); Expr::Literal(Literal::new(token)) } - py_ast::Expr::Constant(cons) => match cons.value { + py_ast::Constant::Float(f) => { + let token = Token::new( + TokenKind::RatioLit, + f.to_string(), + const_.location().row.get(), + const_.location().column.to_zero_indexed(), + ); + Expr::Literal(Literal::new(token)) } - py_ast::Expr::String { value } => { - let StringGroup::Constant{ value } = value else { - return Expr::Dummy(Dummy::new(None, vec![])); - }; + py_ast::Constant::Complex { real: _, imag: _ } => Expr::Dummy(Dummy::new(None, vec![])), + py_ast::Constant::Str(value) => { let value = format!("\"{value}\""); // column - 2 because of the quotes let token = Token::new( TokenKind::StrLit, value, - expr.location().start().row() as u32, - expr.location().start().column() as u32 - 2, + const_.location().row.get(), + const_.location().column.to_zero_indexed() - 1, ); Expr::Literal(Literal::new(token)) } - py_ast::Expr::False => Expr::Literal(Literal::new(Token::new( - TokenKind::BoolLit, - "False", - expr.location().start().row() as u32, - expr.location().start().column() as u32 - 1, - ))), - py_ast::Expr::True => Expr::Literal(Literal::new(Token::new( - TokenKind::BoolLit, - "True", - expr.location().start().row() as u32, - expr.location().start().column() as u32 - 1, - ))), - py_ast::Expr::None => Expr::Literal(Literal::new(Token::new( + py_ast::Constant::Bool(b) => { + let cont = if b { "True" } else { "False" }; + Expr::Literal(Literal::new(Token::new( + TokenKind::BoolLit, + cont, + const_.location().row.get(), + const_.location().column.to_zero_indexed(), + ))) + } + py_ast::Constant::None => Expr::Literal(Literal::new(Token::new( TokenKind::NoneLit, "None", - expr.location().start().row() as u32, - expr.location().start().column() as u32 - 1, + const_.location().row.get(), + const_.location().column.to_zero_indexed(), ))), - py_ast::Expr::Ellipsis => Expr::Literal(Literal::new(Token::new( + py_ast::Constant::Ellipsis => Expr::Literal(Literal::new(Token::new( TokenKind::EllipsisLit, "...", - expr.location().start().row() as u32, - expr.location().start().column() as u32 - 1, + const_.location().row.get(), + const_.location().column.to_zero_indexed(), ))), + // Bytes, Tuple + other => { + log!(err "unknown: {other:?}"); + Expr::Dummy(Dummy::new(None, vec![])) + } + } + } + + fn convert_expr(&mut self, expr: py_ast::Expr) -> Expr { + match expr { + py_ast::Expr::Constant(const_) => self.convert_const(const_), py_ast::Expr::Name(name) => { - let ident = self.convert_ident(name, expr.location().start()); + let ident = self.convert_ident(name.id.to_string(), expr.location()); Expr::Accessor(Accessor::Ident(ident)) } py_ast::Expr::Attribute(attr) => { - let obj = self.convert_expr(*value); + let obj = self.convert_expr(*attr.value); let attr_name_loc = PyLocation::new( obj.ln_end().unwrap_or(1) as usize, obj.col_end().unwrap_or(1) as usize + 2, ); - let name = self.convert_attr_ident(name, attr_name_loc); + let name = self.convert_attr_ident(attr.attr.to_string(), attr_name_loc); obj.attr_expr(name) } py_ast::Expr::IfExp(if_) => { @@ -966,11 +978,13 @@ impl ASTConverter { } py_ast::Expr::Call(call) => { let function = self.convert_expr(*call.func); - let pos_args = call.args + let pos_args = call + .args .into_iter() .map(|ex| PosArg::new(self.convert_expr(ex))) .collect::>(); - let kw_args = call.keywords + let kw_args = call + .keywords .into_iter() .map(|Keyword { arg, value, range }| { let name = arg.unwrap_or_default(); @@ -1002,13 +1016,13 @@ impl ASTConverter { function.call_expr(args) } py_ast::Expr::BinOp(bin) => { - let lhs = self.convert_expr(*a); - let rhs = self.convert_expr(*b); - let op = op_to_token(op); + let lhs = self.convert_expr(*bin.left); + let rhs = self.convert_expr(*bin.right); + let op = op_to_token(bin.op); Expr::BinOp(BinOp::new(op, lhs, rhs)) } py_ast::Expr::UnaryOp(un) => { - let rhs = self.convert_expr(*a); + let rhs = self.convert_expr(*un.operand); let (kind, cont) = match un.op { Unaryop::UAdd => (TokenKind::PrePlus, "+"), // Unaryop::Not => (TokenKind::PreBitNot, "not"), @@ -1059,12 +1073,10 @@ impl ASTConverter { Expr::Lambda(Lambda::new(sig, op, Block::new(body), DefId(0))) } py_ast::Expr::List(list) => { - let (l_sqbr, r_sqbr) = Self::gen_enclosure_tokens( - TokenKind::LSqBr, - list.elts.iter(), - expr.location(), - ); - let elements = list.elts + let (l_sqbr, r_sqbr) = + Self::gen_enclosure_tokens(TokenKind::LSqBr, list.elts.iter(), expr.location()); + let elements = list + .elts .into_iter() .map(|ex| PosArg::new(self.convert_expr(ex))) .collect::>(); @@ -1073,12 +1085,10 @@ impl ASTConverter { Self::mutate_expr(arr) } py_ast::Expr::Set(set) => { - let (l_brace, r_brace) = Self::gen_enclosure_tokens( - TokenKind::LBrace, - set.elts.iter(), - expr.location(), - ); - let elements = set.elts + let (l_brace, r_brace) = + Self::gen_enclosure_tokens(TokenKind::LBrace, set.elts.iter(), expr.location()); + let elements = set + .elts .into_iter() .map(|ex| PosArg::new(self.convert_expr(ex))) .collect::>(); @@ -1092,7 +1102,8 @@ impl ASTConverter { dict.values.iter(), expr.location(), ); - let kvs = dict.keys + let kvs = dict + .keys .into_iter() .zip(dict.values.into_iter()) .map(|(k, v)| { @@ -1107,7 +1118,8 @@ impl ASTConverter { Self::mutate_expr(dict) } py_ast::Expr::Tuple(tuple) => { - let elements = tuple.elts + let elements = tuple + .elts .into_iter() .map(|ex| PosArg::new(self.convert_expr(ex))) .collect::>(); @@ -1115,11 +1127,10 @@ impl ASTConverter { Expr::Tuple(Tuple::Normal(NormalTuple::new(elems))) } py_ast::Expr::Subscript(subs) => { - let obj = self.convert_expr(*a); - let method = obj.attr_expr( - self.convert_ident("__getitem__".to_string(), expr.location()), - ); - method.call1(self.convert_expr(*b)) + let obj = self.convert_expr(*subs.value); + let method = + obj.attr_expr(self.convert_ident("__getitem__".to_string(), expr.location())); + method.call1(self.convert_expr(*subs.slice)) } _other => { log!(err "unimplemented: {:?}", _other); @@ -1266,7 +1277,8 @@ impl ASTConverter { self.namespace.last().unwrap().into(), sig.ln_begin().unwrap_or(0), ); - let class_spec = TypeSpec::mono(class_ident); + let class_ident_expr = Expr::Accessor(Accessor::Ident(class_ident.clone())); + let class_spec = TypeSpecWithOp::new(COLON, TypeSpec::mono(class_ident), class_ident_expr); let sig = Signature::Subr(SubrSignature::new( set! { Decorator(Expr::static_local("Override")) }, call_ident, @@ -1290,7 +1302,8 @@ impl ASTConverter { let params = Params::new(vec![], None, vec![], None); let class_ident = Identifier::public_with_line(DOT, self.namespace.last().unwrap().into(), line as u32); - let class_spec = TypeSpec::mono(class_ident); + let class_ident_expr = Expr::Accessor(Accessor::Ident(class_ident.clone())); + let class_spec = TypeSpecWithOp::new(COLON, TypeSpec::mono(class_ident), class_ident_expr); let sig = Signature::Subr(SubrSignature::new( set! { Decorator(Expr::static_local("Override")) }, call_ident, @@ -1433,11 +1446,20 @@ impl ASTConverter { .map(|ex| Decorator(self.convert_expr(ex))) .collect::>(); self.register_name_info(&name, NameKind::Function); - let func_name_loc = PyLocation{ row: loc.row.get(), column: loc.column.get() + 4 }; + let func_name_loc = PyLocation { + row: loc.row, + column: loc.column.saturating_add(4), + }; let ident = self.convert_ident(name, func_name_loc); self.namespace.push(ident.inspect().to_string()); let params = self.convert_params(params); - let return_t = returns.map(|ret| self.convert_type_spec(ret)); + let return_t = returns.map(|ret| { + TypeSpecWithOp::new( + COLON, + self.convert_type_spec(ret.clone()), + self.convert_expr(ret), + ) + }); let sig = Signature::Subr(SubrSignature::new( decos, ident, @@ -1485,7 +1507,10 @@ impl ASTConverter { .collect::>(); let inherit = !bases.is_empty(); self.register_name_info(&name, NameKind::Class); - let class_name_loc = PyLocation::new(loc.row(), loc.column() + 6); + let class_name_loc = PyLocation { + row: loc.row, + column: loc.column.saturating_add(6), + }; let ident = self.convert_ident(name, class_name_loc); let sig = Signature::Var(VarSignature::new(VarPattern::Ident(ident.clone()), None)); self.namespace.push(ident.inspect().to_string()); @@ -1550,16 +1575,17 @@ impl ASTConverter { Expr::Def(def) } else { // no registration because it's just a type ascription - let ident = self.convert_ident(name, stmt.location()); + let ident = self.convert_ident(name.id.to_string(), stmt.location()); let tasc = TypeAscription::new(Expr::Accessor(Accessor::Ident(ident)), t_spec); Expr::TypeAscription(tasc) } } py_ast::Expr::Attribute(attr) => { - let attr = self - .convert_expr(*attr.value) - .attr(self.convert_attr_ident(attr.attr, ann_assign.target.location())); + let attr = self.convert_expr(*attr.value).attr(self.convert_attr_ident( + attr.attr.to_string(), + ann_assign.target.location(), + )); if let Some(value) = ann_assign.value { let expr = self.convert_expr(*value); let redef = ReDef::new(attr, expr); @@ -1579,8 +1605,8 @@ impl ASTConverter { py_ast::Expr::Name(name) => { let block = Block::new(vec![self.convert_expr(*assign.value)]); let body = DefBody::new(EQUAL, block, DefId(0)); - self.register_name_info(&name, NameKind::Variable); - let ident = self.convert_ident(name, stmt.location); + self.register_name_info(&name.id, NameKind::Variable); + let ident = self.convert_ident(name.id.to_string(), stmt.location()); let sig = Signature::Var(VarSignature::new(VarPattern::Ident(ident), None)); let def = Def::new(sig, body); @@ -1663,8 +1689,9 @@ impl ASTConverter { py_ast::Expr::Name(name) => { let body = DefBody::new(EQUAL, Block::new(vec![value.clone()]), DefId(0)); - self.register_name_info(&name, NameKind::Variable); - let ident = self.convert_ident(name, stmt.location()); + self.register_name_info(&name.id, NameKind::Variable); + let ident = + self.convert_ident(name.id.to_string(), stmt.location()); let sig = Signature::Var(VarSignature::new( VarPattern::Ident(ident), None, @@ -1682,17 +1709,17 @@ impl ASTConverter { } py_ast::Stmt::AugAssign(aug_assign) => { let op = op_to_token(aug_assign.op); - match aug_assign.target { - py_ast::Expr::Identifier { name } => { - let val = self.convert_expr(*value); - let prev_ident = self.convert_ident(name.clone(), stmt.location()); + match *aug_assign.target { + py_ast::Expr::Name(name) => { + let val = self.convert_expr(*aug_assign.value); + let prev_ident = self.convert_ident(name.id.to_string(), stmt.location()); if self - .get_name(&name) + .get_name(name.id.as_str()) .map(|info| info.defined_block_id == self.cur_block_id()) .unwrap_or(false) { - self.register_name_info(&name, NameKind::Variable); - let ident = self.convert_ident(name.clone(), stmt.location()); + self.register_name_info(&name.id, NameKind::Variable); + let ident = self.convert_ident(name.id.to_string(), stmt.location()); let bin = BinOp::new(op, Expr::Accessor(Accessor::Ident(prev_ident)), val); let sig = @@ -1702,7 +1729,7 @@ impl ASTConverter { let def = Def::new(sig, body); Expr::Def(def) } else { - let ident = self.convert_ident(name.clone(), stmt.location()); + let ident = self.convert_ident(name.id.to_string(), stmt.location()); let bin = BinOp::new(op, Expr::Accessor(Accessor::Ident(prev_ident)), val); let redef = ReDef::new(Accessor::Ident(ident), Expr::BinOp(bin)); @@ -1710,10 +1737,10 @@ impl ASTConverter { } } py_ast::Expr::Attribute(attr) => { - let val = self.convert_expr(*value); + let val = self.convert_expr(*aug_assign.value); let attr = self .convert_expr(*attr) - .attr(self.convert_attr_ident(name, target.location)); + .attr(self.convert_attr_ident(name, aug_assign.target.location())); let bin = BinOp::new(op, Expr::Accessor(attr.clone()), val); let adef = ReDef::new(attr, Expr::BinOp(bin)); Expr::ReDef(adef) @@ -1724,8 +1751,21 @@ impl ASTConverter { } } } - py_ast::Stmt::FunctionDef(func_def) => self.convert_funcdef(name, *args, body, decorator_list, returns, range), - py_ast::Stmt::ClassDef(class_def) => self.convert_classdef(name, body, bases, decorator_list, range), + py_ast::Stmt::FunctionDef(func_def) => self.convert_funcdef( + func_def.name.to_string(), + *func_def.args, + func_def.body, + func_def.decorator_list, + func_def.returns.map(|x| *x), + func_def.range, + ), + py_ast::Stmt::ClassDef(class_def) => self.convert_classdef( + class_def.name.to_string(), + class_def.body, + class_def.bases, + class_def.decorator_list, + class_def.location(), + ), py_ast::Stmt::For(for_) => { let iter = self.convert_expr(*for_.iter); let block = self.convert_for_body(Some(*for_.target), for_.body); @@ -1769,8 +1809,9 @@ impl ASTConverter { } } py_ast::Stmt::Return(return_) => { - let value = return_.value - .map(|val| self.convert_expr(val)) + let value = return_ + .value + .map(|val| self.convert_expr(*val)) .unwrap_or_else(|| Expr::Tuple(Tuple::Normal(NormalTuple::new(Args::empty())))); if dont_call_return { value @@ -1813,7 +1854,9 @@ impl ASTConverter { let def = if let Some(alias) = name.asname { self.register_name_info(&alias, NameKind::Variable); let var = VarSignature::new( - VarPattern::Ident(self.convert_ident(alias, stmt.location())), + VarPattern::Ident( + self.convert_ident(alias.to_string(), stmt.location()), + ), None, ); Def::new( @@ -1823,7 +1866,9 @@ impl ASTConverter { } else { self.register_name_info(&name.name, NameKind::Variable); let var = VarSignature::new( - VarPattern::Ident(self.convert_ident(name.name, stmt.location())), + VarPattern::Ident( + self.convert_ident(name.name.to_string(), stmt.location()), + ), None, ); Def::new( @@ -1836,9 +1881,13 @@ impl ASTConverter { Expr::Dummy(Dummy::new(None, imports)) } // from module import foo, bar - py_ast::Stmt::ImportFrom(import_from) => self.convert_from_import(module, names, stmt.location), + py_ast::Stmt::ImportFrom(import_from) => self.convert_from_import( + import_from.module, + import_from.names, + import_from.location(), + ), py_ast::Stmt::Try(try_) => { - let chunks = self.convert_block(body, BlockKind::Try).into_iter(); + let chunks = self.convert_block(try_.body, BlockKind::Try).into_iter(); let dummy = match (orelse, finalbody) { (Some(orelse), Some(finalbody)) => chunks .chain(self.convert_block(orelse, BlockKind::Try).into_iter()) @@ -1857,7 +1906,7 @@ impl ASTConverter { py_ast::Stmt::With(with) => { let item = with.items.remove(0); let context_expr = self.convert_expr(item.context_expr); - let body = self.convert_for_body(item.optional_vars, body); + let body = self.convert_for_body(item.optional_vars.map(|x| *x), with.body); let with_ident = self.convert_ident("with".to_string(), stmt.location()); let with_acc = Expr::Accessor(Accessor::Ident(with_ident)); with_acc.call2(context_expr, Expr::Lambda(body)) @@ -1893,8 +1942,8 @@ impl ASTConverter { */ fn convert_from_import( &mut self, - module: Option, - names: Vec, + module: Option, + names: Vec, location: PyLocation, ) -> Expr { let import_acc = Expr::Accessor(Accessor::Ident( @@ -1912,20 +1961,21 @@ impl ASTConverter { let mod_name = Expr::Literal(Literal::new(Token::new( TokenKind::StrLit, cont, - location.row() as u32, - location.column() as u32 - 1, + location.row.get(), + location.column.to_zero_indexed(), ))); let call = import_acc.clone().call1(mod_name); let mut exprs = vec![]; let mut imports = vec![]; - // `from module import ` - let mut loc = PyLocation::new(location.row(), location.column() + 5 + module.len() + 8); for name in names { - let name_path = self.cfg.input.resolve_py(&module_path.join(&name.symbol)); - let true_name = self.convert_ident(name.symbol.clone(), loc); - let alias = if let Some(alias) = name.alias { + let name_path = self + .cfg + .input + .resolve_py(&module_path.join(name.name.as_str())); + let true_name = self.convert_ident(name.name.to_string(), name.location()); + let alias = if let Some(alias) = name.asname { // ` as ` - for _ in 0..name.symbol.len() + 4 { + for _ in 0..name.name.len() + 4 { loc.go_right(); } self.register_name_info(&alias, NameKind::Variable); @@ -1937,21 +1987,21 @@ impl ASTConverter { } VarSignature::new(VarPattern::Ident(ident), None) } else { - self.register_name_info(&name.symbol, NameKind::Variable); - let ident = self.convert_ident(name.symbol.clone(), loc); - for _ in 0..name.symbol.len() + 2 { + self.register_name_info(&name.name, NameKind::Variable); + let ident = self.convert_ident(name.name.to_string(), name.location()); + for _ in 0..name.name.len() + 2 { loc.go_right(); } VarSignature::new(VarPattern::Ident(ident), None) }; // from foo import bar, baz (if bar, baz is a module) ==> bar = import "foo/bar"; baz = import "foo/baz" if let Ok(_path) = name_path { - let cont = format!("\"{module}/{}\"", name.symbol); + let cont = format!("\"{module}/{}\"", name.name); let mod_name = Expr::Literal(Literal::new(Token::new( TokenKind::StrLit, cont, - location.row() as u32, - location.column() as u32 - 1, + location.row.get(), + location.column.to_zero_indexed(), ))); let call = import_acc.clone().call1(mod_name); let def = Def::new( diff --git a/src/analyze.rs b/src/analyze.rs index b8086ff..906b22b 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -14,7 +14,7 @@ use erg_compiler::error::{CompileError, CompileErrors}; use erg_compiler::lower::ASTLowerer; use erg_compiler::module::SharedCompilerResource; use py2erg::{dump_decl_er, reserve_decl_er, ShadowingMode}; -use rustpython_parser::{Parse, ast as py_ast}; +use rustpython_parser::{ast as py_ast, Parse}; use crate::handle_err; From 39c27524ec490ab86ec887604dcfd8ce802003ab Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Wed, 7 Jun 2023 17:09:03 +0900 Subject: [PATCH 4/8] WIP --- Cargo.lock | 2 + Cargo.toml | 4 +- crates/py2erg/Cargo.toml | 1 + crates/py2erg/ast_util.rs | 2 +- crates/py2erg/convert.rs | 266 ++++++++++++++++++++------------------ src/analyze.rs | 9 +- 6 files changed, 151 insertions(+), 133 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb84057..61f75bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -548,6 +548,7 @@ version = "0.0.29" dependencies = [ "erg_common", "erg_compiler", + "rustpython-ast", "rustpython-parser", ] @@ -559,6 +560,7 @@ dependencies = [ "erg_common", "erg_compiler", "py2erg", + "rustpython-ast", "rustpython-parser", ] diff --git a/Cargo.toml b/Cargo.toml index 8b7ed51..a2392f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,8 @@ repository = "https://github.com/mtshiba/pylyzer" erg_common = { version = "0.6.14", features = ["py_compat", "els"] } erg_compiler = { version = "0.6.14", features = ["py_compat", "els"] } els = { version = "0.1.26", features = ["py_compat"] } -rustpython-parser = { git = "https://github.com/RustPython/Parser", version = "0.2.0", features = ["all-nodes-with-ranges"] } +rustpython-parser = { git = "https://github.com/RustPython/Parser", version = "0.2.0", features = ["all-nodes-with-ranges", "location"] } +rustpython-ast = { git = "https://github.com/RustPython/Parser", version = "0.2.0", features = ["all-nodes-with-ranges", "location"] } # erg_compiler = { git = "https://github.com/erg-lang/erg", branch = "main", features = ["py_compat", "els"] } # erg_common = { git = "https://github.com/erg-lang/erg", branch = "main", features = ["py_compat", "els"] } # els = { git = "https://github.com/erg-lang/erg", branch = "main", features = ["py_compat"] } @@ -44,6 +45,7 @@ erg_compiler = { workspace = true } erg_common = { workspace = true } els = { workspace = true } rustpython-parser = { workspace = true } +rustpython-ast = { workspace = true } py2erg = { version = "0.0.29", path = "./crates/py2erg" } [lib] diff --git a/crates/py2erg/Cargo.toml b/crates/py2erg/Cargo.toml index 77471b6..94461c3 100644 --- a/crates/py2erg/Cargo.toml +++ b/crates/py2erg/Cargo.toml @@ -14,6 +14,7 @@ debug = ["erg_compiler/debug", "erg_common/debug"] [dependencies] rustpython-parser = { workspace = true } +rustpython-ast = { workspace = true } erg_common = { workspace = true } erg_compiler = { workspace = true } diff --git a/crates/py2erg/ast_util.rs b/crates/py2erg/ast_util.rs index 5dec411..e70e656 100644 --- a/crates/py2erg/ast_util.rs +++ b/crates/py2erg/ast_util.rs @@ -1,4 +1,4 @@ -use rustpython_parser::ast::Expr; +use rustpython_parser::ast::located::Expr; pub fn accessor_name(expr: Expr) -> Option { match expr { diff --git a/crates/py2erg/convert.rs b/crates/py2erg/convert.rs index 895b079..3261a50 100644 --- a/crates/py2erg/convert.rs +++ b/crates/py2erg/convert.rs @@ -26,7 +26,7 @@ use rustpython_parser::ast::located::{ self as py_ast, Alias, Arg, Arguments, Boolop, Cmpop, ExprConstant, Keyword, Located, ModModule, Operator, Stmt, String, Suite, Unaryop, }; -use rustpython_parser::source_code::{SourceLocation as PyLocation, SourceRange as PySourceRange}; +use rustpython_parser::source_code::{SourceLocation as PyLocation, SourceRange as PySourceRange, OneIndexed}; use crate::ast_util::accessor_name; use crate::error::*; @@ -383,18 +383,26 @@ impl ASTConverter { } fn convert_params(&mut self, params: Arguments) -> Params { - let non_defaults_len = params.args.len() - params.defaults(); - let mut args = params.into_python_arguments(); - let defaults = params.defaults(); - let non_defaults = args - .args + fn split_kwonlyargs(params: Arguments) -> (Vec, Vec<(Arg, py_ast::Expr)>) { + let mut args = Vec::new(); + let mut with_defaults = Vec::new(); + for arg in params.kwonlyargs.into_iter() { + if let Some(default) = arg.default { + with_defaults.push((arg.def, *default)); + } else { + args.push(arg.def); + } + } + (args, with_defaults) + } + let (non_defaults, defaults) = split_kwonlyargs(params); + let non_defaults = non_defaults .into_iter() .map(|p| self.convert_nd_param(p)) .collect(); - let defaults = defaults_names + let defaults = defaults .into_iter() - .zip(params.defaults()) - .map(|(kw, default)| self.convert_default_param(kw, default.clone())) + .map(|(kw, default)| self.convert_default_param(kw, default)) .collect(); Params::new(non_defaults, None, defaults, None) } @@ -441,6 +449,7 @@ impl ASTConverter { vec![], ), py_ast::Expr::Tuple(expr) => { + let loc = expr.location(); let tmp = fresh_varname(); let tmp_name = VarName::from_str_and_line((&tmp).into(), expr.location().row.get()); let tmp_expr = Expr::Accessor(Accessor::Ident(Identifier::new( @@ -453,7 +462,7 @@ impl ASTConverter { TokenKind::NatLit, i.to_string(), elem.location().row.get(), - elem.location().column.get() as u32 - 1, + elem.location().column.to_zero_indexed(), )); let (param, mut blocks) = self.convert_expr_to_param(elem); let sig = Signature::Var(VarSignature::new( @@ -462,7 +471,7 @@ impl ASTConverter { )); let method = tmp_expr .clone() - .attr_expr(self.convert_ident("__getitem__".to_string(), expr.location())); + .attr_expr(self.convert_ident("__getitem__".to_string(), loc)); let tuple_acc = method.call1(Expr::Literal(index)); let body = DefBody::new(EQUAL, Block::new(vec![tuple_acc]), DefId(0)); let def = Expr::Def(Def::new(sig, body)); @@ -472,12 +481,12 @@ impl ASTConverter { let pat = ParamPattern::VarName(tmp_name); (NonDefaultParamSignature::new(pat, None), block) } - _other => { + other => { let token = Token::new( TokenKind::UBar, "_", - expr.location().row.get(), - expr.location().column.get() - 1, + other.location().row.get(), + other.location().column.to_zero_indexed(), ); ( NonDefaultParamSignature::new(ParamPattern::Discard(token), None), @@ -520,7 +529,7 @@ impl ASTConverter { fn convert_compound_type_spec(&mut self, name: String, args: py_ast::Expr) -> TypeSpec { match &name[..] { "Union" => { - let py_ast::Expr::Tuple(tuple) = args else { + let py_ast::Expr::Tuple(mut tuple) = args else { let err = CompileError::syntax_error( self.cfg.input.clone(), line!() as usize, @@ -574,7 +583,7 @@ impl ASTConverter { } // TODO: distinguish from collections.abc.Callable "Callable" => { - let py_ast::Expr::Tuple(tuple) = args else { + let py_ast::Expr::Tuple(mut tuple) = args else { return Self::gen_dummy_type_spec(args.location()); }; let params = tuple.elts.remove(0); @@ -590,7 +599,7 @@ impl ASTConverter { let err = CompileError::syntax_error( self.cfg.input.clone(), line!() as usize, - pyloc_to_ergloc(params.range()), + pyloc_to_ergloc(other.range()), self.cur_namespace(), "Expected a list of parameters".into(), None, @@ -634,7 +643,7 @@ impl ASTConverter { TypeSpec::poly(acc, ConstArgs::pos_only(vec![elem_t], None)) } "Mapping" | "MutableMapping" => { - let py_ast::Expr::Tuple(tuple) = args else { + let py_ast::Expr::Tuple(mut tuple) = args else { let err = CompileError::syntax_error( self.cfg.input.clone(), line!() as usize, @@ -710,13 +719,13 @@ impl ASTConverter { TypeSpec::poly(acc, ConstArgs::new(vec![elem_t, len], None, vec![], None)) } "dict" => { - let py_ast::Expr::Tuple(tuple) = args else { + let py_ast::Expr::Tuple(mut tuple) = args else { return Self::gen_dummy_type_spec(args.location()); }; let (l_brace, r_brace) = Self::gen_enclosure_tokens( TokenKind::LBrace, tuple.elts.iter(), - args.location(), + tuple.location(), ); let key_t = self.convert_expr(tuple.elts.remove(0)); let key_t = match Parser::validate_const_expr(key_t) { @@ -764,7 +773,7 @@ impl ASTConverter { let parens = Self::gen_enclosure_tokens( TokenKind::LParen, tuple.elts.iter(), - args.location(), + tuple.location(), ); let tys = tuple .elts @@ -782,14 +791,15 @@ impl ASTConverter { #[allow(clippy::collapsible_match)] match expr { py_ast::Expr::Name(name) => { - self.convert_ident_type_spec(name.id.to_string(), expr.location()) + self.convert_ident_type_spec(name.id.to_string(), name.location()) } py_ast::Expr::Constant(cons) => { - self.convert_ident_type_spec("NoneType".into(), expr.location()) + self.convert_ident_type_spec("NoneType".into(), cons.location()) } py_ast::Expr::Attribute(attr) => { + let loc = attr.location(); let namespace = Box::new(self.convert_expr(*attr.value)); - let t = self.convert_ident(attr.attr.to_string(), expr.location()); + let t = self.convert_ident(attr.attr.to_string(), loc); let predecl = PreDeclTypeSpec::Attr { namespace, t }; TypeSpec::PreDeclTy(predecl) } @@ -799,13 +809,14 @@ impl ASTConverter { self.convert_compound_type_spec(name.id.to_string(), *subs.slice) } py_ast::Expr::Attribute(attr) => { - match accessor_name(attr.value).as_ref().map(|s| &s[..]) { + let loc = attr.location(); + match accessor_name(*attr.value).as_ref().map(|s| &s[..]) { Some("typing" | "collections.abc") => { self.convert_compound_type_spec(attr.attr.to_string(), *subs.slice) } - _ => { - log!(err "unknown: {subs:?}"); - Self::gen_dummy_type_spec(subs.value.location()) + other => { + log!(err "unknown: {other:?}"); + Self::gen_dummy_type_spec(loc) } } } @@ -815,6 +826,7 @@ impl ASTConverter { } }, py_ast::Expr::BinOp(bin) => { + let loc = bin.location(); match bin.op { // A | B Operator::BitOr => { @@ -822,12 +834,12 @@ impl ASTConverter { let rhs = self.convert_type_spec(*bin.right); TypeSpec::or(lhs, rhs) } - _ => Self::gen_dummy_type_spec(expr.location()), + _ => Self::gen_dummy_type_spec(loc), } } other => { log!(err "unknown: {other:?}"); - Self::gen_dummy_type_spec(expr.location()) + Self::gen_dummy_type_spec(other.location()) } } } @@ -858,7 +870,7 @@ impl ASTConverter { expr_loc.row.get(), expr_loc.column.get() - 1, ); - let r_brace = Token::new(r_kind, r_cont, l_end as u32, c_end as u32); + let r_brace = Token::new(r_kind, r_cont, l_end, c_end); (l_brace, r_brace) } @@ -873,6 +885,7 @@ impl ASTConverter { } fn convert_const(&mut self, const_: ExprConstant) -> Expr { + let loc = const_.location(); match const_.value { py_ast::Constant::Int(i) => { let kind = if i >= 0.into() { @@ -883,8 +896,8 @@ impl ASTConverter { let token = Token::new( kind, i.to_string(), - const_.location().row.get(), - const_.location().column.to_zero_indexed(), + loc.row.get(), + loc.column.to_zero_indexed(), ); Expr::Literal(Literal::new(token)) } @@ -904,8 +917,8 @@ impl ASTConverter { let token = Token::new( TokenKind::StrLit, value, - const_.location().row.get(), - const_.location().column.to_zero_indexed() - 1, + loc.row.get(), + loc.column.to_zero_indexed() - 1, ); Expr::Literal(Literal::new(token)) } @@ -914,8 +927,8 @@ impl ASTConverter { Expr::Literal(Literal::new(Token::new( TokenKind::BoolLit, cont, - const_.location().row.get(), - const_.location().column.to_zero_indexed(), + loc.row.get(), + loc.column.to_zero_indexed(), ))) } py_ast::Constant::None => Expr::Literal(Literal::new(Token::new( @@ -942,25 +955,26 @@ impl ASTConverter { match expr { py_ast::Expr::Constant(const_) => self.convert_const(const_), py_ast::Expr::Name(name) => { - let ident = self.convert_ident(name.id.to_string(), expr.location()); + let ident = self.convert_ident(name.id.to_string(), name.location()); Expr::Accessor(Accessor::Ident(ident)) } py_ast::Expr::Attribute(attr) => { let obj = self.convert_expr(*attr.value); - let attr_name_loc = PyLocation::new( - obj.ln_end().unwrap_or(1) as usize, - obj.col_end().unwrap_or(1) as usize + 2, - ); + let attr_name_loc = PyLocation { + row: OneIndexed::new(obj.ln_end().unwrap_or(1)).unwrap(), + column: OneIndexed::new(obj.col_end().unwrap_or(1) + 2).unwrap(), + }; let name = self.convert_attr_ident(attr.attr.to_string(), attr_name_loc); obj.attr_expr(name) } py_ast::Expr::IfExp(if_) => { + let loc = if_.location(); let block = self.convert_expr(*if_.body); let params = Params::new(vec![], None, vec![], None); let sig = LambdaSignature::new(params.clone(), None, TypeBoundSpecs::empty()); let body = Lambda::new(sig, Token::DUMMY, Block::new(vec![block]), DefId(0)); let test = self.convert_expr(*if_.test); - let if_ident = self.convert_ident("if".to_string(), expr.location()); + let if_ident = self.convert_ident("if".to_string(), loc); let if_acc = Expr::Accessor(Accessor::Ident(if_ident)); let else_block = self.convert_expr(*if_.orelse); let sig = LambdaSignature::new(params, None, TypeBoundSpecs::empty()); @@ -977,6 +991,7 @@ impl ASTConverter { if_acc.call_expr(args) } py_ast::Expr::Call(call) => { + let loc = call.location(); let function = self.convert_expr(*call.func); let pos_args = call .args @@ -987,8 +1002,8 @@ impl ASTConverter { .keywords .into_iter() .map(|Keyword { arg, value, range }| { - let name = arg.unwrap_or_default(); - let name = Token::symbol_with_loc(name.as_str(), pyloc_to_ergloc(range)); + let name = arg.unwrap(); + let name = Token::symbol_with_loc(name.to_string(), pyloc_to_ergloc(range)); let ex = self.convert_expr(value); KwArg::new(name, None, ex) }) @@ -1001,13 +1016,13 @@ impl ASTConverter { let lp = Token::new( TokenKind::LParen, "(", - expr.location().row.get(), + loc.row.get(), function.col_end().unwrap_or(0) + 1, ); let rp = Token::new( TokenKind::RParen, ")", - expr.location().row.get(), + loc.row.get(), last_col + 1, ); (lp, rp) @@ -1034,10 +1049,10 @@ impl ASTConverter { Expr::UnaryOp(UnaryOp::new(op, rhs)) } // TODO - py_ast::Expr::BoolOp(bool) => { - let lhs = self.convert_expr(bool.values.remove(0)); - let rhs = self.convert_expr(bool.values.remove(0)); - let (kind, cont) = match bool.op { + py_ast::Expr::BoolOp(mut boole) => { + let lhs = self.convert_expr(boole.values.remove(0)); + let rhs = self.convert_expr(boole.values.remove(0)); + let (kind, cont) = match boole.op { Boolop::And => (TokenKind::AndOp, "and"), Boolop::Or => (TokenKind::OrOp, "or"), }; @@ -1045,7 +1060,7 @@ impl ASTConverter { Expr::BinOp(BinOp::new(op, lhs, rhs)) } // TODO: multiple Cmpops - py_ast::Expr::Compare(cmp) => { + py_ast::Expr::Compare(mut cmp) => { let lhs = self.convert_expr(*cmp.left); let rhs = self.convert_expr(cmp.comparators.remove(0)); let (kind, cont) = match cmp.ops.remove(0) { @@ -1074,7 +1089,7 @@ impl ASTConverter { } py_ast::Expr::List(list) => { let (l_sqbr, r_sqbr) = - Self::gen_enclosure_tokens(TokenKind::LSqBr, list.elts.iter(), expr.location()); + Self::gen_enclosure_tokens(TokenKind::LSqBr, list.elts.iter(), list.location()); let elements = list .elts .into_iter() @@ -1086,7 +1101,7 @@ impl ASTConverter { } py_ast::Expr::Set(set) => { let (l_brace, r_brace) = - Self::gen_enclosure_tokens(TokenKind::LBrace, set.elts.iter(), expr.location()); + Self::gen_enclosure_tokens(TokenKind::LBrace, set.elts.iter(), set.location()); let elements = set .elts .into_iter() @@ -1100,7 +1115,7 @@ impl ASTConverter { let (l_brace, r_brace) = Self::gen_enclosure_tokens( TokenKind::LBrace, dict.values.iter(), - expr.location(), + dict.location(), ); let kvs = dict .keys @@ -1129,7 +1144,7 @@ impl ASTConverter { py_ast::Expr::Subscript(subs) => { let obj = self.convert_expr(*subs.value); let method = - obj.attr_expr(self.convert_ident("__getitem__".to_string(), expr.location())); + obj.attr_expr(self.convert_ident("__getitem__".to_string(), subs.slice.location())); method.call1(self.convert_expr(*subs.slice)) } _other => { @@ -1566,7 +1581,7 @@ impl ASTConverter { let body = DefBody::new(EQUAL, block, DefId(0)); // must register after convert_expr because value may be contain name (e.g. i = i + 1) self.register_name_info(name.id.as_str(), NameKind::Variable); - let ident = self.convert_ident(name.id.to_string(), stmt.location()); + let ident = self.convert_ident(name.id.to_string(), name.location()); let sig = Signature::Var(VarSignature::new( VarPattern::Ident(ident), Some(t_spec), @@ -1575,16 +1590,17 @@ impl ASTConverter { Expr::Def(def) } else { // no registration because it's just a type ascription - let ident = self.convert_ident(name.id.to_string(), stmt.location()); + let ident = self.convert_ident(name.id.to_string(), name.location()); let tasc = TypeAscription::new(Expr::Accessor(Accessor::Ident(ident)), t_spec); Expr::TypeAscription(tasc) } } py_ast::Expr::Attribute(attr) => { + let loc = attr.location(); let attr = self.convert_expr(*attr.value).attr(self.convert_attr_ident( attr.attr.to_string(), - ann_assign.target.location(), + loc, )); if let Some(value) = ann_assign.value { let expr = self.convert_expr(*value); @@ -1598,7 +1614,7 @@ impl ASTConverter { _other => Expr::Dummy(Dummy::new(None, vec![])), } } - py_ast::Stmt::Assign(assign) => { + py_ast::Stmt::Assign(mut assign) => { if assign.targets.len() == 1 { let lhs = assign.targets.remove(0); match lhs { @@ -1606,17 +1622,17 @@ impl ASTConverter { let block = Block::new(vec![self.convert_expr(*assign.value)]); let body = DefBody::new(EQUAL, block, DefId(0)); self.register_name_info(&name.id, NameKind::Variable); - let ident = self.convert_ident(name.id.to_string(), stmt.location()); + let ident = self.convert_ident(name.id.to_string(), name.location()); let sig = Signature::Var(VarSignature::new(VarPattern::Ident(ident), None)); let def = Def::new(sig, body); Expr::Def(def) } py_ast::Expr::Attribute(attr) => { - let attr_name_loc = attr.end_location(); + let attr_name_loc = attr.end_location().unwrap_or(attr.location()); let attr = self .convert_expr(*attr.value) - .attr(self.convert_attr_ident(name, attr_name_loc)); + .attr(self.convert_attr_ident(attr.attr.to_string(), attr_name_loc)); let expr = self.convert_expr(*assign.value); let adef = ReDef::new(attr, expr); Expr::ReDef(adef) @@ -1625,7 +1641,7 @@ impl ASTConverter { let tmp = fresh_varname(); let tmp_name = VarName::from_str_and_line( (&tmp).into(), - stmt.location().row.get(), + tuple.location().row.get(), ); let tmp_ident = Identifier::new(VisModifierSpec::Public(DOT), tmp_name); let tmp_expr = Expr::Accessor(Accessor::Ident(tmp_ident.clone())); @@ -1641,11 +1657,12 @@ impl ASTConverter { let tmp_def = Expr::Def(Def::new(sig, body)); let mut defs = vec![tmp_def]; for (i, elem) in tuple.elts.into_iter().enumerate() { + let loc = elem.location(); let index = Literal::new(Token::new( TokenKind::NatLit, i.to_string(), - elem.location().row.get(), - elem.location().column.get() - 1, + loc.row.get(), + loc.column.to_zero_indexed(), )); let (param, mut blocks) = self.convert_opt_expr_to_param(Some(elem)); @@ -1654,7 +1671,7 @@ impl ASTConverter { param.t_spec, )); let method = tmp_expr.clone().attr_expr( - self.convert_ident("__getitem__".to_string(), stmt.location()), + self.convert_ident("__getitem__".to_string(), loc), ); let tuple_acc = method.call1(Expr::Literal(index)); let body = @@ -1669,10 +1686,11 @@ impl ASTConverter { // => a.__setitem__(b, x) py_ast::Expr::Subscript(subs) => { let a = self.convert_expr(*subs.value); + let slice_loc = subs.slice.location(); let b = self.convert_expr(*subs.slice); let x = self.convert_expr(*assign.value); let method = a.attr_expr( - self.convert_ident("__setitem__".to_string(), stmt.location()), + self.convert_ident("__setitem__".to_string(), slice_loc), ); method.call2(b, x) } @@ -1691,7 +1709,7 @@ impl ASTConverter { DefBody::new(EQUAL, Block::new(vec![value.clone()]), DefId(0)); self.register_name_info(&name.id, NameKind::Variable); let ident = - self.convert_ident(name.id.to_string(), stmt.location()); + self.convert_ident(name.id.to_string(), name.location()); let sig = Signature::Var(VarSignature::new( VarPattern::Ident(ident), None, @@ -1712,14 +1730,14 @@ impl ASTConverter { match *aug_assign.target { py_ast::Expr::Name(name) => { let val = self.convert_expr(*aug_assign.value); - let prev_ident = self.convert_ident(name.id.to_string(), stmt.location()); + let prev_ident = self.convert_ident(name.id.to_string(), name.location()); if self .get_name(name.id.as_str()) .map(|info| info.defined_block_id == self.cur_block_id()) .unwrap_or(false) { self.register_name_info(&name.id, NameKind::Variable); - let ident = self.convert_ident(name.id.to_string(), stmt.location()); + let ident = self.convert_ident(name.id.to_string(), name.location()); let bin = BinOp::new(op, Expr::Accessor(Accessor::Ident(prev_ident)), val); let sig = @@ -1729,7 +1747,7 @@ impl ASTConverter { let def = Def::new(sig, body); Expr::Def(def) } else { - let ident = self.convert_ident(name.id.to_string(), stmt.location()); + let ident = self.convert_ident(name.id.to_string(), name.location()); let bin = BinOp::new(op, Expr::Accessor(Accessor::Ident(prev_ident)), val); let redef = ReDef::new(Accessor::Ident(ident), Expr::BinOp(bin)); @@ -1738,9 +1756,10 @@ impl ASTConverter { } py_ast::Expr::Attribute(attr) => { let val = self.convert_expr(*aug_assign.value); + let attr_loc = attr.location(); let attr = self - .convert_expr(*attr) - .attr(self.convert_attr_ident(name, aug_assign.target.location())); + .convert_expr(*attr.value) + .attr(self.convert_attr_ident(attr.attr.to_string(), attr_loc)); let bin = BinOp::new(op, Expr::Accessor(attr.clone()), val); let adef = ReDef::new(attr, Expr::BinOp(bin)); Expr::ReDef(adef) @@ -1759,40 +1778,46 @@ impl ASTConverter { func_def.returns.map(|x| *x), func_def.range, ), - py_ast::Stmt::ClassDef(class_def) => self.convert_classdef( - class_def.name.to_string(), - class_def.body, - class_def.bases, - class_def.decorator_list, - class_def.location(), - ), + py_ast::Stmt::ClassDef(class_def) => { + let class_loc = class_def.location(); + self.convert_classdef( + class_def.name.to_string(), + class_def.body, + class_def.bases, + class_def.decorator_list, + class_loc, + ) + }, py_ast::Stmt::For(for_) => { + let loc = for_.location(); let iter = self.convert_expr(*for_.iter); let block = self.convert_for_body(Some(*for_.target), for_.body); - let for_ident = self.convert_ident("for".to_string(), stmt.location()); + let for_ident = self.convert_ident("for".to_string(), loc); let for_acc = Expr::Accessor(Accessor::Ident(for_ident)); for_acc.call2(iter, Expr::Lambda(block)) } py_ast::Stmt::While(while_) => { + let loc = while_.location(); let test = self.convert_expr(*while_.test); let params = Params::new(vec![], None, vec![], None); let empty_sig = LambdaSignature::new(params, None, TypeBoundSpecs::empty()); let block = self.convert_block(while_.body, BlockKind::While); let body = Lambda::new(empty_sig, Token::DUMMY, block, DefId(0)); - let while_ident = self.convert_ident("while".to_string(), stmt.location()); + let while_ident = self.convert_ident("while".to_string(), loc); let while_acc = Expr::Accessor(Accessor::Ident(while_ident)); while_acc.call2(test, Expr::Lambda(body)) } py_ast::Stmt::If(if_) => { + let loc = if_.location(); let block = self.convert_block(if_.body, BlockKind::If); let params = Params::new(vec![], None, vec![], None); let sig = LambdaSignature::new(params.clone(), None, TypeBoundSpecs::empty()); let body = Lambda::new(sig, Token::DUMMY, block, DefId(0)); let test = self.convert_expr(*if_.test); - let if_ident = self.convert_ident("if".to_string(), stmt.location()); + let if_ident = self.convert_ident("if".to_string(), loc); let if_acc = Expr::Accessor(Accessor::Ident(if_ident)); - if let Some(orelse) = if_.orelse { - let else_block = self.convert_block(orelse, BlockKind::If); + if !if_.orelse.is_empty() { + let else_block = self.convert_block(if_.orelse, BlockKind::If); let sig = LambdaSignature::new(params, None, TypeBoundSpecs::empty()); let else_body = Lambda::new(sig, Token::DUMMY, else_block, DefId(0)); let args = Args::pos_only( @@ -1809,6 +1834,7 @@ impl ASTConverter { } } py_ast::Stmt::Return(return_) => { + let loc = return_.location(); let value = return_ .value .map(|val| self.convert_expr(*val)) @@ -1817,14 +1843,15 @@ impl ASTConverter { value } else { let func_acc = Expr::Accessor(Accessor::Ident( - self.convert_ident(self.namespace.last().unwrap().clone(), stmt.location()), + self.convert_ident(self.namespace.last().unwrap().clone(), loc), )); - let return_acc = self.convert_ident("return".to_string(), stmt.location()); + let return_acc = self.convert_ident("return".to_string(), loc); let return_acc = Expr::Accessor(Accessor::attr(func_acc, return_acc)); return_acc.call1(value) } } py_ast::Stmt::Assert(assert) => { + let loc = assert.location(); let test = self.convert_expr(*assert.test); let args = if let Some(msg) = assert.msg { let msg = self.convert_expr(*msg); @@ -1833,29 +1860,31 @@ impl ASTConverter { Args::pos_only(vec![PosArg::new(test)], None) }; let assert_acc = Expr::Accessor(Accessor::Ident( - self.convert_ident("assert".to_string(), stmt.location()), + self.convert_ident("assert".to_string(), loc), )); assert_acc.call_expr(args) } py_ast::Stmt::Import(import) => { + let loc = import.location(); let mut imports = vec![]; for name in import.names { let import_acc = Expr::Accessor(Accessor::Ident( - self.convert_ident("__import__".to_string(), stmt.location()), + self.convert_ident("__import__".to_string(), loc), )); let cont = format!("\"{}\"", name.name.split('.').next().unwrap()); let mod_name = Expr::Literal(Literal::new(Token::new( TokenKind::StrLit, cont, - stmt.location().row.get(), - stmt.location().column.get() - 1, + name.location().row.get(), + name.location().column.get() - 1, ))); let call = import_acc.call1(mod_name); + let loc = name.location(); let def = if let Some(alias) = name.asname { self.register_name_info(&alias, NameKind::Variable); let var = VarSignature::new( VarPattern::Ident( - self.convert_ident(alias.to_string(), stmt.location()), + self.convert_ident(alias.to_string(), loc), ), None, ); @@ -1867,7 +1896,7 @@ impl ASTConverter { self.register_name_info(&name.name, NameKind::Variable); let var = VarSignature::new( VarPattern::Ident( - self.convert_ident(name.name.to_string(), stmt.location()), + self.convert_ident(name.name.to_string(), name.location()), ), None, ); @@ -1881,33 +1910,27 @@ impl ASTConverter { Expr::Dummy(Dummy::new(None, imports)) } // from module import foo, bar - py_ast::Stmt::ImportFrom(import_from) => self.convert_from_import( - import_from.module, - import_from.names, - import_from.location(), - ), + py_ast::Stmt::ImportFrom(import_from) => { + let loc = import_from.location(); + self.convert_from_import( + import_from.module, + import_from.names, + loc, + ) + }, py_ast::Stmt::Try(try_) => { let chunks = self.convert_block(try_.body, BlockKind::Try).into_iter(); - let dummy = match (orelse, finalbody) { - (Some(orelse), Some(finalbody)) => chunks - .chain(self.convert_block(orelse, BlockKind::Try).into_iter()) - .chain(self.convert_block(finalbody, BlockKind::Try).into_iter()) - .collect(), - (Some(orelse), None) => chunks - .chain(self.convert_block(orelse, BlockKind::Try).into_iter()) - .collect(), - (None, Some(finalbody)) => chunks - .chain(self.convert_block(finalbody, BlockKind::Try).into_iter()) - .collect(), - (None, None) => chunks.collect(), - }; + let dummy = chunks.chain(self.convert_block(try_.orelse, BlockKind::Try)) + .chain(self.convert_block(try_.finalbody, BlockKind::Try)) + .collect(); Expr::Dummy(Dummy::new(None, dummy)) } - py_ast::Stmt::With(with) => { + py_ast::Stmt::With(mut with) => { + let loc = with.location(); let item = with.items.remove(0); let context_expr = self.convert_expr(item.context_expr); let body = self.convert_for_body(item.optional_vars.map(|x| *x), with.body); - let with_ident = self.convert_ident("with".to_string(), stmt.location()); + let with_ident = self.convert_ident("with".to_string(), loc); let with_acc = Expr::Accessor(Accessor::Ident(with_ident)); with_acc.call2(context_expr, Expr::Lambda(body)) } @@ -1973,25 +1996,14 @@ impl ASTConverter { .input .resolve_py(&module_path.join(name.name.as_str())); let true_name = self.convert_ident(name.name.to_string(), name.location()); + let as_loc = name.location(); let alias = if let Some(alias) = name.asname { - // ` as ` - for _ in 0..name.name.len() + 4 { - loc.go_right(); - } self.register_name_info(&alias, NameKind::Variable); - let alias_len = alias.len(); - let ident = self.convert_ident(alias, loc); - // `, ` - for _ in 0..alias_len + 2 { - loc.go_right(); - } + let ident = self.convert_ident(alias.to_string(), as_loc); VarSignature::new(VarPattern::Ident(ident), None) } else { self.register_name_info(&name.name, NameKind::Variable); let ident = self.convert_ident(name.name.to_string(), name.location()); - for _ in 0..name.name.len() + 2 { - loc.go_right(); - } VarSignature::new(VarPattern::Ident(ident), None) }; // from foo import bar, baz (if bar, baz is a module) ==> bar = import "foo/bar"; baz = import "foo/baz" diff --git a/src/analyze.rs b/src/analyze.rs index 906b22b..6783866 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -14,7 +14,8 @@ use erg_compiler::error::{CompileError, CompileErrors}; use erg_compiler::lower::ASTLowerer; use erg_compiler::module::SharedCompilerResource; use py2erg::{dump_decl_er, reserve_decl_er, ShadowingMode}; -use rustpython_parser::{ast as py_ast, Parse}; +use rustpython_parser::Parse; +use rustpython_ast::located::ModModule; use crate::handle_err; @@ -22,7 +23,7 @@ pub struct SimplePythonParser {} impl Parsable for SimplePythonParser { fn parse(code: String) -> Result { - let py_program = py_ast::Suite::parse(&code).map_err(|_err| ParseErrors::empty())?; + let py_program = ModModule::parse(&code, "").map_err(|_err| ParseErrors::empty())?; // TODO: SourceLocator let shadowing = if cfg!(feature = "debug") { ShadowingMode::Visible @@ -110,13 +111,13 @@ impl PythonAnalyzer { mode: &str, ) -> Result { let filename = self.cfg.input.filename(); - let py_program = parser::parse_program(&py_code).map_err(|err| { + let py_program = ModModule::parse(&py_code, &filename).map_err(|err| { let core = ErrorCore::new( vec![], err.to_string(), 0, ErrorKind::SyntaxError, - erg_common::error::Location::Line(err.location.row() as u32), + erg_common::error::Location::Line(todo!()), ); let err = CompileError::new(core, self.cfg.input.clone(), "".into()); IncompleteArtifact::new(None, CompileErrors::from(err), CompileErrors::empty()) From d36d4150bd7ed3d7a2271ee9d9f4d43ecad78bff Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Thu, 8 Jun 2023 01:53:00 +0900 Subject: [PATCH 5/8] Update analyze.rs --- src/analyze.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/analyze.rs b/src/analyze.rs index 6783866..6f8e700 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -14,8 +14,9 @@ use erg_compiler::error::{CompileError, CompileErrors}; use erg_compiler::lower::ASTLowerer; use erg_compiler::module::SharedCompilerResource; use py2erg::{dump_decl_er, reserve_decl_er, ShadowingMode}; -use rustpython_parser::Parse; -use rustpython_ast::located::ModModule; +use rustpython_ast::source_code::{LinearLocator}; +use rustpython_parser::{Parse, ParseErrorType}; +use rustpython_ast::{Fold, ModModule}; use crate::handle_err; @@ -24,7 +25,8 @@ pub struct SimplePythonParser {} impl Parsable for SimplePythonParser { fn parse(code: String) -> Result { let py_program = ModModule::parse(&code, "").map_err(|_err| ParseErrors::empty())?; - // TODO: SourceLocator + let mut locator = LinearLocator::new(&code); + let py_program = locator.fold(py_program).map_err(|_err| ParseErrors::empty())?; let shadowing = if cfg!(feature = "debug") { ShadowingMode::Visible } else { @@ -112,16 +114,22 @@ impl PythonAnalyzer { ) -> Result { let filename = self.cfg.input.filename(); let py_program = ModModule::parse(&py_code, &filename).map_err(|err| { + let mut locator = LinearLocator::new(&py_code); + let err = locator.locate_error::<_, ParseErrorType>(err); + let msg = err.to_string(); + let loc = err.location.unwrap(); let core = ErrorCore::new( vec![], - err.to_string(), + msg, 0, ErrorKind::SyntaxError, - erg_common::error::Location::Line(todo!()), + erg_common::error::Location::range(loc.row.get(), loc.column.to_zero_indexed(), loc.row.get(), loc.column.to_zero_indexed()), ); let err = CompileError::new(core, self.cfg.input.clone(), "".into()); IncompleteArtifact::new(None, CompileErrors::from(err), CompileErrors::empty()) })?; + let mut locator = LinearLocator::new(&py_code); + let py_program = locator.fold(py_program).unwrap(); let shadowing = if cfg!(feature = "debug") { ShadowingMode::Visible } else { From 25336075c1691f1184c30d347260592cd8ebd9f4 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Thu, 8 Jun 2023 02:38:40 +0900 Subject: [PATCH 6/8] Update convert.rs --- crates/py2erg/convert.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/py2erg/convert.rs b/crates/py2erg/convert.rs index 3261a50..8cad8a1 100644 --- a/crates/py2erg/convert.rs +++ b/crates/py2erg/convert.rs @@ -1871,7 +1871,11 @@ impl ASTConverter { let import_acc = Expr::Accessor(Accessor::Ident( self.convert_ident("__import__".to_string(), loc), )); - let cont = format!("\"{}\"", name.name.split('.').next().unwrap()); + let cont = if name.asname.is_some() { + format!("\"{}\"", name.name.replace('.', "/")) + } else { + format!("\"{}\"", name.name.split('.').next().unwrap()) + }; let mod_name = Expr::Literal(Literal::new(Token::new( TokenKind::StrLit, cont, From df54032e5413b09dcc01cb0f04cad7f459cf6a29 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Fri, 9 Jun 2023 21:06:13 +0900 Subject: [PATCH 7/8] build: update deps --- Cargo.lock | 16 ++++----- Cargo.toml | 6 ++-- crates/py2erg/convert.rs | 71 ++++++++++++++++++++------------------- crates/py2erg/error.rs | 2 +- crates/py2erg/gen_decl.rs | 2 +- src/analyze.rs | 52 ++++++++++++++++++---------- src/config.rs | 5 +-- tests/test.rs | 3 +- 8 files changed, 89 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61f75bb..1829427 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -152,9 +152,9 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "els" -version = "0.1.26" +version = "0.1.27-nightly.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad69bdb783d8f54317278b3d32b138b05fe0708c9a22454cd9505f39edf4d044" +checksum = "52d65a32e855c6093747f38e0c2d4049b0f38c4b74dea213bf5031b18ebca141" dependencies = [ "erg_common", "erg_compiler", @@ -177,18 +177,18 @@ dependencies = [ [[package]] name = "erg_common" -version = "0.6.14" +version = "0.6.15-nightly.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abef65c68abb0bbf94d43bdedddc44349ff02f8a21b510dedf66f4be24587a0c" +checksum = "0fa2ce07958c54ec955967e0008f0aaa016122c1ec18b30ba2ddeffa9f61b3f2" dependencies = [ "backtrace-on-stack-overflow", ] [[package]] name = "erg_compiler" -version = "0.6.14" +version = "0.6.15-nightly.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09c82d01efbb07c5da0d5ee35ce6fc053e09512dca0275136e5cf2c07bd7ee43" +checksum = "e136c552b1707b4c9d158a636ab1a3f09e2c4dee6b5d511623998080ed6021f9" dependencies = [ "erg_common", "erg_parser", @@ -196,9 +196,9 @@ dependencies = [ [[package]] name = "erg_parser" -version = "0.6.14" +version = "0.6.15-nightly.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b572d85a2a33d729a8dd03a655ee1c36b9e86f4e85deb22f9043542e4b8fb41f" +checksum = "ad991bc363694adf11d621278109ec82d5328e2191b7f369dcb66b78f85b413a" dependencies = [ "erg_common", "unicode-xid", diff --git a/Cargo.toml b/Cargo.toml index a2392f1..67afc36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,9 +22,9 @@ edition = "2021" repository = "https://github.com/mtshiba/pylyzer" [workspace.dependencies] -erg_common = { version = "0.6.14", features = ["py_compat", "els"] } -erg_compiler = { version = "0.6.14", features = ["py_compat", "els"] } -els = { version = "0.1.26", features = ["py_compat"] } +erg_common = { version = "0.6.15-nightly.0", features = ["py_compat", "els"] } +erg_compiler = { version = "0.6.15-nightly.0", features = ["py_compat", "els"] } +els = { version = "0.1.27-nightly.0", features = ["py_compat"] } rustpython-parser = { git = "https://github.com/RustPython/Parser", version = "0.2.0", features = ["all-nodes-with-ranges", "location"] } rustpython-ast = { git = "https://github.com/RustPython/Parser", version = "0.2.0", features = ["all-nodes-with-ranges", "location"] } # erg_compiler = { git = "https://github.com/erg-lang/erg", branch = "main", features = ["py_compat", "els"] } diff --git a/crates/py2erg/convert.rs b/crates/py2erg/convert.rs index 8cad8a1..3a2a705 100644 --- a/crates/py2erg/convert.rs +++ b/crates/py2erg/convert.rs @@ -26,7 +26,9 @@ use rustpython_parser::ast::located::{ self as py_ast, Alias, Arg, Arguments, Boolop, Cmpop, ExprConstant, Keyword, Located, ModModule, Operator, Stmt, String, Suite, Unaryop, }; -use rustpython_parser::source_code::{SourceLocation as PyLocation, SourceRange as PySourceRange, OneIndexed}; +use rustpython_parser::source_code::{ + OneIndexed, SourceLocation as PyLocation, SourceRange as PySourceRange, +}; use crate::ast_util::accessor_name; use crate::error::*; @@ -383,10 +385,15 @@ impl ASTConverter { } fn convert_params(&mut self, params: Arguments) -> Params { - fn split_kwonlyargs(params: Arguments) -> (Vec, Vec<(Arg, py_ast::Expr)>) { + fn split_args(params: Arguments) -> (Vec, Vec<(Arg, py_ast::Expr)>) { let mut args = Vec::new(); let mut with_defaults = Vec::new(); - for arg in params.kwonlyargs.into_iter() { + for arg in params + .posonlyargs + .into_iter() + .chain(params.args.into_iter()) + .chain(params.kwonlyargs.into_iter()) + { if let Some(default) = arg.default { with_defaults.push((arg.def, *default)); } else { @@ -395,7 +402,7 @@ impl ASTConverter { } (args, with_defaults) } - let (non_defaults, defaults) = split_kwonlyargs(params); + let (non_defaults, defaults) = split_args(params); let non_defaults = non_defaults .into_iter() .map(|p| self.convert_nd_param(p)) @@ -1019,12 +1026,7 @@ impl ASTConverter { loc.row.get(), function.col_end().unwrap_or(0) + 1, ); - let rp = Token::new( - TokenKind::RParen, - ")", - loc.row.get(), - last_col + 1, - ); + let rp = Token::new(TokenKind::RParen, ")", loc.row.get(), last_col + 1); (lp, rp) }; let args = Args::new(pos_args, None, kw_args, Some(paren)); @@ -1143,8 +1145,9 @@ impl ASTConverter { } py_ast::Expr::Subscript(subs) => { let obj = self.convert_expr(*subs.value); - let method = - obj.attr_expr(self.convert_ident("__getitem__".to_string(), subs.slice.location())); + let method = obj.attr_expr( + self.convert_ident("__getitem__".to_string(), subs.slice.location()), + ); method.call1(self.convert_expr(*subs.slice)) } _other => { @@ -1573,7 +1576,13 @@ impl ASTConverter { py_ast::Stmt::AnnAssign(ann_assign) => { let anot = self.convert_expr(*ann_assign.annotation.clone()); let t_spec = self.convert_type_spec(*ann_assign.annotation); - let t_spec = TypeSpecWithOp::new(AS, t_spec, anot); + let as_op = Token::new( + TokenKind::As, + "as", + t_spec.ln_begin().unwrap_or(0), + t_spec.col_begin().unwrap_or(0), + ); + let t_spec = TypeSpecWithOp::new(as_op, t_spec, anot); match *ann_assign.target { py_ast::Expr::Name(name) => { if let Some(value) = ann_assign.value { @@ -1598,10 +1607,9 @@ impl ASTConverter { } py_ast::Expr::Attribute(attr) => { let loc = attr.location(); - let attr = self.convert_expr(*attr.value).attr(self.convert_attr_ident( - attr.attr.to_string(), - loc, - )); + let attr = self + .convert_expr(*attr.value) + .attr(self.convert_attr_ident(attr.attr.to_string(), loc)); if let Some(value) = ann_assign.value { let expr = self.convert_expr(*value); let redef = ReDef::new(attr, expr); @@ -1630,9 +1638,9 @@ impl ASTConverter { } py_ast::Expr::Attribute(attr) => { let attr_name_loc = attr.end_location().unwrap_or(attr.location()); - let attr = self - .convert_expr(*attr.value) - .attr(self.convert_attr_ident(attr.attr.to_string(), attr_name_loc)); + let attr = self.convert_expr(*attr.value).attr( + self.convert_attr_ident(attr.attr.to_string(), attr_name_loc), + ); let expr = self.convert_expr(*assign.value); let adef = ReDef::new(attr, expr); Expr::ReDef(adef) @@ -1670,9 +1678,9 @@ impl ASTConverter { Self::param_pattern_to_var(param.pat), param.t_spec, )); - let method = tmp_expr.clone().attr_expr( - self.convert_ident("__getitem__".to_string(), loc), - ); + let method = tmp_expr + .clone() + .attr_expr(self.convert_ident("__getitem__".to_string(), loc)); let tuple_acc = method.call1(Expr::Literal(index)); let body = DefBody::new(EQUAL, Block::new(vec![tuple_acc]), DefId(0)); @@ -1787,7 +1795,7 @@ impl ASTConverter { class_def.decorator_list, class_loc, ) - }, + } py_ast::Stmt::For(for_) => { let loc = for_.location(); let iter = self.convert_expr(*for_.iter); @@ -1887,9 +1895,7 @@ impl ASTConverter { let def = if let Some(alias) = name.asname { self.register_name_info(&alias, NameKind::Variable); let var = VarSignature::new( - VarPattern::Ident( - self.convert_ident(alias.to_string(), loc), - ), + VarPattern::Ident(self.convert_ident(alias.to_string(), loc)), None, ); Def::new( @@ -1916,15 +1922,12 @@ impl ASTConverter { // from module import foo, bar py_ast::Stmt::ImportFrom(import_from) => { let loc = import_from.location(); - self.convert_from_import( - import_from.module, - import_from.names, - loc, - ) - }, + self.convert_from_import(import_from.module, import_from.names, loc) + } py_ast::Stmt::Try(try_) => { let chunks = self.convert_block(try_.body, BlockKind::Try).into_iter(); - let dummy = chunks.chain(self.convert_block(try_.orelse, BlockKind::Try)) + let dummy = chunks + .chain(self.convert_block(try_.orelse, BlockKind::Try)) .chain(self.convert_block(try_.finalbody, BlockKind::Try)) .collect(); Expr::Dummy(Dummy::new(None, dummy)) diff --git a/crates/py2erg/error.rs b/crates/py2erg/error.rs index fd411b1..a402b9d 100644 --- a/crates/py2erg/error.rs +++ b/crates/py2erg/error.rs @@ -1,5 +1,5 @@ -use erg_common::config::Input; use erg_common::error::{ErrorCore, ErrorKind, Location, SubMessage}; +use erg_common::io::Input; use erg_common::switch_lang; use erg_compiler::error::CompileError; diff --git a/crates/py2erg/gen_decl.rs b/crates/py2erg/gen_decl.rs index f006722..0886567 100644 --- a/crates/py2erg/gen_decl.rs +++ b/crates/py2erg/gen_decl.rs @@ -1,7 +1,7 @@ use std::fs::File; use std::io::{BufWriter, Write}; -use erg_common::config::Input; +use erg_common::io::Input; use erg_common::log; use erg_compiler::context::register::{CheckStatus, PylyzerStatus}; use erg_compiler::hir::{Expr, HIR}; diff --git a/src/analyze.rs b/src/analyze.rs index 6f8e700..15125ea 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -8,36 +8,45 @@ use erg_compiler::artifact::{BuildRunnable, Buildable, CompleteArtifact, Incompl use erg_compiler::context::register::CheckStatus; use erg_compiler::context::ModuleContext; use erg_compiler::erg_parser::ast::{Module, AST}; -use erg_compiler::erg_parser::error::ParseErrors; +use erg_compiler::erg_parser::error::{ + CompleteArtifact as ParseArtifact, IncompleteArtifact as IncompleteParseArtifact, ParseErrors, +}; use erg_compiler::erg_parser::parse::Parsable; use erg_compiler::error::{CompileError, CompileErrors}; use erg_compiler::lower::ASTLowerer; use erg_compiler::module::SharedCompilerResource; use py2erg::{dump_decl_er, reserve_decl_er, ShadowingMode}; -use rustpython_ast::source_code::{LinearLocator}; -use rustpython_parser::{Parse, ParseErrorType}; +use rustpython_ast::source_code::RandomLocator; use rustpython_ast::{Fold, ModModule}; +use rustpython_parser::{Parse, ParseErrorType}; use crate::handle_err; pub struct SimplePythonParser {} impl Parsable for SimplePythonParser { - fn parse(code: String) -> Result { + fn parse(code: String) -> Result> { let py_program = ModModule::parse(&code, "").map_err(|_err| ParseErrors::empty())?; - let mut locator = LinearLocator::new(&code); - let py_program = locator.fold(py_program).map_err(|_err| ParseErrors::empty())?; + let mut locator = RandomLocator::new(&code); + // let mut locator = LinearLocator::new(&code); + let py_program = locator + .fold(py_program) + .map_err(|_err| ParseErrors::empty())?; let shadowing = if cfg!(feature = "debug") { ShadowingMode::Visible } else { ShadowingMode::Invisible }; let converter = py2erg::ASTConverter::new(ErgConfig::default(), shadowing); - let IncompleteArtifact{ object: Some(erg_module), errors, .. } = converter.convert_program(py_program) else { unreachable!() }; + let IncompleteArtifact{ object: Some(erg_module), errors, warns } = converter.convert_program(py_program) else { unreachable!() }; if errors.is_empty() { - Ok(erg_module) + Ok(ParseArtifact::new(erg_module, warns.into())) } else { - Err(ParseErrors::empty()) + Err(IncompleteParseArtifact::new( + Some(erg_module), + warns.into(), + errors.into(), + )) } } } @@ -114,7 +123,8 @@ impl PythonAnalyzer { ) -> Result { let filename = self.cfg.input.filename(); let py_program = ModModule::parse(&py_code, &filename).map_err(|err| { - let mut locator = LinearLocator::new(&py_code); + let mut locator = RandomLocator::new(&py_code); + // let mut locator = LinearLocator::new(&py_code); let err = locator.locate_error::<_, ParseErrorType>(err); let msg = err.to_string(); let loc = err.location.unwrap(); @@ -123,12 +133,18 @@ impl PythonAnalyzer { msg, 0, ErrorKind::SyntaxError, - erg_common::error::Location::range(loc.row.get(), loc.column.to_zero_indexed(), loc.row.get(), loc.column.to_zero_indexed()), + erg_common::error::Location::range( + loc.row.get(), + loc.column.to_zero_indexed(), + loc.row.get(), + loc.column.to_zero_indexed(), + ), ); let err = CompileError::new(core, self.cfg.input.clone(), "".into()); IncompleteArtifact::new(None, CompileErrors::from(err), CompileErrors::empty()) })?; - let mut locator = LinearLocator::new(&py_code); + let mut locator = RandomLocator::new(&py_code); + // let mut locator = LinearLocator::new(&py_code); let py_program = locator.fold(py_program).unwrap(); let shadowing = if cfg!(feature = "debug") { ShadowingMode::Visible @@ -165,7 +181,7 @@ impl PythonAnalyzer { } pub fn run(&mut self) { - if self.cfg.output_dir.is_some() { + if self.cfg.dist_dir.is_some() { reserve_decl_er(self.cfg.input.clone()); } let py_code = self.cfg.input.read(); @@ -179,13 +195,13 @@ impl PythonAnalyzer { artifact.warns.len(), self.cfg.input.unescaped_filename() ); - artifact.warns.fmt_all_stderr(); + artifact.warns.write_all_stderr(); } println!( "{GREEN}All checks OK{RESET}: {}", self.cfg.input.unescaped_filename() ); - if self.cfg.output_dir.is_some() { + if self.cfg.dist_dir.is_some() { dump_decl_er( self.cfg.input.clone(), artifact.object, @@ -202,7 +218,7 @@ impl PythonAnalyzer { artifact.warns.len(), self.cfg.input.unescaped_filename() ); - artifact.warns.fmt_all_stderr(); + artifact.warns.write_all_stderr(); } let code = if artifact.errors.is_empty() { println!( @@ -216,11 +232,11 @@ impl PythonAnalyzer { artifact.errors.len(), self.cfg.input.unescaped_filename() ); - artifact.errors.fmt_all_stderr(); + artifact.errors.write_all_stderr(); 1 }; // Even if type checking fails, some APIs are still valid, so generate a file - if self.cfg.output_dir.is_some() { + if self.cfg.dist_dir.is_some() { dump_decl_er( self.cfg.input.clone(), artifact.object.unwrap(), diff --git a/src/config.rs b/src/config.rs index 6451fc4..fd85ba8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,7 +2,8 @@ use std::env; use std::path::PathBuf; use std::str::FromStr; -use erg_common::config::{ErgConfig, ErgMode, Input}; +use erg_common::config::{ErgConfig, ErgMode}; +use erg_common::io::Input; use erg_common::switch_lang; fn command_message() -> &'static str { @@ -94,7 +95,7 @@ pub(crate) fn parse_args() -> ErgConfig { cfg.quiet_repl = true; } "--dump-decl" => { - cfg.output_dir = Some(""); + cfg.dist_dir = Some(""); } "--verbose" => { cfg.verbose = args diff --git a/tests/test.rs b/tests/test.rs index e785d7a..e87058f 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -1,6 +1,7 @@ use std::path::PathBuf; -use erg_common::config::{ErgConfig, Input}; +use erg_common::config::ErgConfig; +use erg_common::io::Input; use erg_common::spawn::exec_new_thread; use erg_common::traits::Stream; use erg_compiler::artifact::{CompleteArtifact, IncompleteArtifact}; From b9839553b13327b1e56b8859d9199ba976fab48c Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Fri, 9 Jun 2023 21:54:02 +0900 Subject: [PATCH 8/8] Update convert.rs --- crates/py2erg/convert.rs | 95 ++++++++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 38 deletions(-) diff --git a/crates/py2erg/convert.rs b/crates/py2erg/convert.rs index 3a2a705..b7ac54e 100644 --- a/crates/py2erg/convert.rs +++ b/crates/py2erg/convert.rs @@ -19,7 +19,7 @@ use erg_compiler::erg_parser::ast::{ VisModifierSpec, }; use erg_compiler::erg_parser::desugar::Desugarer; -use erg_compiler::erg_parser::token::{Token, TokenKind, AS, COLON, DOT, EQUAL}; +use erg_compiler::erg_parser::token::{Token, TokenKind, COLON, DOT, EQUAL}; use erg_compiler::erg_parser::Parser; use erg_compiler::error::{CompileError, CompileErrors}; use rustpython_parser::ast::located::{ @@ -121,12 +121,19 @@ fn op_to_token(op: Operator) -> Token { pub fn pyloc_to_ergloc(range: PySourceRange) -> erg_common::error::Location { erg_common::error::Location::range( range.start.row.get(), - range.start.column.get(), + range.start.column.to_zero_indexed(), range.end.unwrap().row.get(), range.end.unwrap().column.get(), ) } +fn attr_name_loc(value: &Expr) -> PyLocation { + PyLocation { + row: OneIndexed::from_zero_indexed(value.ln_end().unwrap_or(0)).saturating_sub(1), + column: OneIndexed::from_zero_indexed(value.col_end().unwrap_or(0)).saturating_add(1), + } +} + #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum DefinedPlace { Known(String), @@ -374,7 +381,15 @@ impl ASTConverter { self.convert_expr(*anot), ) }) - .map(|(t_spec, expr)| TypeSpecWithOp::new(AS, t_spec, expr)); + .map(|(t_spec, expr)| { + let as_op = Token::new( + TokenKind::As, + "as", + t_spec.ln_begin().unwrap_or(0), + t_spec.col_begin().unwrap_or(0), + ); + TypeSpecWithOp::new(as_op, t_spec, expr) + }); NonDefaultParamSignature::new(pat, t_spec) } @@ -804,9 +819,8 @@ impl ASTConverter { self.convert_ident_type_spec("NoneType".into(), cons.location()) } py_ast::Expr::Attribute(attr) => { - let loc = attr.location(); let namespace = Box::new(self.convert_expr(*attr.value)); - let t = self.convert_ident(attr.attr.to_string(), loc); + let t = self.convert_ident(attr.attr.to_string(), attr_name_loc(&namespace)); let predecl = PreDeclTypeSpec::Attr { namespace, t }; TypeSpec::PreDeclTy(predecl) } @@ -925,7 +939,7 @@ impl ASTConverter { TokenKind::StrLit, value, loc.row.get(), - loc.column.to_zero_indexed() - 1, + loc.column.to_zero_indexed(), ); Expr::Literal(Literal::new(token)) } @@ -966,13 +980,9 @@ impl ASTConverter { Expr::Accessor(Accessor::Ident(ident)) } py_ast::Expr::Attribute(attr) => { - let obj = self.convert_expr(*attr.value); - let attr_name_loc = PyLocation { - row: OneIndexed::new(obj.ln_end().unwrap_or(1)).unwrap(), - column: OneIndexed::new(obj.col_end().unwrap_or(1) + 2).unwrap(), - }; - let name = self.convert_attr_ident(attr.attr.to_string(), attr_name_loc); - obj.attr_expr(name) + let value = self.convert_expr(*attr.value); + let name = self.convert_attr_ident(attr.attr.to_string(), attr_name_loc(&value)); + value.attr_expr(name) } py_ast::Expr::IfExp(if_) => { let loc = if_.location(); @@ -1018,15 +1028,15 @@ impl ASTConverter { let last_col = pos_args .last() .and_then(|last| last.col_end()) - .unwrap_or(function.col_end().unwrap_or(0)); + .unwrap_or(function.col_end().unwrap_or(0) + 1); let paren = { let lp = Token::new( TokenKind::LParen, "(", loc.row.get(), - function.col_end().unwrap_or(0) + 1, + function.col_end().unwrap_or(0), ); - let rp = Token::new(TokenKind::RParen, ")", loc.row.get(), last_col + 1); + let rp = Token::new(TokenKind::RParen, ")", loc.row.get(), last_col); (lp, rp) }; let args = Args::new(pos_args, None, kw_args, Some(paren)); @@ -1246,7 +1256,13 @@ impl ASTConverter { ); let param_typ_spec = TypeSpec::mono(param_typ_ident.clone()); let expr = Expr::Accessor(Accessor::Ident(param_typ_ident.clone())); - let param_typ_spec = TypeSpecWithOp::new(AS, param_typ_spec, expr); + let as_op = Token::new( + TokenKind::As, + "as", + param_typ_spec.ln_begin().unwrap_or(0), + param_typ_spec.col_begin().unwrap_or(0), + ); + let param_typ_spec = TypeSpecWithOp::new(as_op, param_typ_spec, expr); let arg_typ_ident = Identifier::public_with_line( DOT, arg_typ_name.into(), @@ -1472,11 +1488,14 @@ impl ASTConverter { self.namespace.push(ident.inspect().to_string()); let params = self.convert_params(params); let return_t = returns.map(|ret| { - TypeSpecWithOp::new( - COLON, - self.convert_type_spec(ret.clone()), - self.convert_expr(ret), - ) + let t_spec = self.convert_type_spec(ret.clone()); + let colon = Token::new( + TokenKind::Colon, + ":", + t_spec.ln_begin().unwrap_or(0), + t_spec.col_begin().unwrap_or(0), + ); + TypeSpecWithOp::new(colon, t_spec, self.convert_expr(ret)) }); let sig = Signature::Subr(SubrSignature::new( decos, @@ -1606,10 +1625,10 @@ impl ASTConverter { } } py_ast::Expr::Attribute(attr) => { - let loc = attr.location(); - let attr = self - .convert_expr(*attr.value) - .attr(self.convert_attr_ident(attr.attr.to_string(), loc)); + let value = self.convert_expr(*attr.value); + let ident = + self.convert_attr_ident(attr.attr.to_string(), attr_name_loc(&value)); + let attr = value.attr(ident); if let Some(value) = ann_assign.value { let expr = self.convert_expr(*value); let redef = ReDef::new(attr, expr); @@ -1637,10 +1656,10 @@ impl ASTConverter { Expr::Def(def) } py_ast::Expr::Attribute(attr) => { - let attr_name_loc = attr.end_location().unwrap_or(attr.location()); - let attr = self.convert_expr(*attr.value).attr( - self.convert_attr_ident(attr.attr.to_string(), attr_name_loc), - ); + let value = self.convert_expr(*attr.value); + let ident = self + .convert_attr_ident(attr.attr.to_string(), attr_name_loc(&value)); + let attr = value.attr(ident); let expr = self.convert_expr(*assign.value); let adef = ReDef::new(attr, expr); Expr::ReDef(adef) @@ -1763,14 +1782,14 @@ impl ASTConverter { } } py_ast::Expr::Attribute(attr) => { - let val = self.convert_expr(*aug_assign.value); - let attr_loc = attr.location(); - let attr = self - .convert_expr(*attr.value) - .attr(self.convert_attr_ident(attr.attr.to_string(), attr_loc)); - let bin = BinOp::new(op, Expr::Accessor(attr.clone()), val); - let adef = ReDef::new(attr, Expr::BinOp(bin)); - Expr::ReDef(adef) + let assign_value = self.convert_expr(*aug_assign.value); + let attr_value = self.convert_expr(*attr.value); + let ident = self + .convert_attr_ident(attr.attr.to_string(), attr_name_loc(&attr_value)); + let attr = attr_value.attr(ident); + let bin = BinOp::new(op, Expr::Accessor(attr.clone()), assign_value); + let redef = ReDef::new(attr, Expr::BinOp(bin)); + Expr::ReDef(redef) } other => { log!(err "{other:?} as LHS");