Try out rust-python-parser (nom-based)

Cargo.lock

@@ -16,7 +16,7 @@ checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
 dependencies = [
  "getrandom 0.2.7",
  "once_cell",
- "version_check",
+ "version_check 0.9.4",
 ]
 
 [[package]]
@@ -319,6 +319,12 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
 
+[[package]]
+name = "bytecount"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f861d9ce359f56dbcb6e0c2a1cb84e52ad732cadb57b806adeb3c7668caccbd8"
+
 [[package]]
 name = "byteorder"
 version = "1.4.3"
@@ -907,7 +913,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
 dependencies = [
  "typenum",
- "version_check",
+ "version_check 0.9.4",
 ]
 
 [[package]]
@@ -1137,7 +1143,7 @@ dependencies = [
  "string_cache",
  "term",
  "tiny-keccak",
- "unicode-xid",
+ "unicode-xid 0.2.4",
 ]
 
 [[package]]
@@ -1366,6 +1372,16 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "nom"
+version = "4.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
+dependencies = [
+ "memchr",
+ "version_check 0.1.5",
+]
+
 [[package]]
 name = "nom"
 version = "5.1.2"
@@ -1373,7 +1389,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
 dependencies = [
  "memchr",
- "version_check",
+ "version_check 0.9.4",
+]
+
+[[package]]
+name = "nom_locate"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a47c112b3861d81f7fbf73892b9271af933af32bd5dee6889aa3c3fa9caed7e"
+dependencies = [
+ "bytecount",
+ "memchr",
+ "nom 4.2.3",
 ]
 
 [[package]]
@@ -1394,6 +1421,17 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "num-bigint"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-bigint"
 version = "0.4.3"
@@ -1695,7 +1733,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "syn",
- "version_check",
+ "version_check 0.9.4",
 ]
 
 [[package]]
@@ -1706,7 +1744,7 @@ checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
 dependencies = [
  "proc-macro2",
  "quote",
- "version_check",
+ "version_check 0.9.4",
 ]
 
 [[package]]
@@ -1718,6 +1756,21 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "python-parser"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683594d14f9c4db9e899643108918ad9db99aedad77121b5179aa205343c10de"
+dependencies = [
+ "nom 4.2.3",
+ "nom_locate",
+ "num-bigint 0.2.6",
+ "num-traits",
+ "unicode-xid 0.1.0",
+ "unicode_names2 0.2.2",
+ "wtf8",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.21"
@@ -1928,6 +1981,7 @@ dependencies = [
  "notify",
  "once_cell",
  "path-absolutize",
+ "python-parser",
  "rayon",
  "regex",
  "rustpython-ast",
@@ -1957,7 +2011,7 @@ name = "rustpython-ast"
 version = "0.1.0"
 source = "git+https://github.com/charliermarsh/RustPython.git?rev=4f457893efc381ad5c432576b24bcc7e4a08c641#4f457893efc381ad5c432576b24bcc7e4a08c641"
 dependencies = [
- "num-bigint",
+ "num-bigint 0.4.3",
  "rustpython-common",
  "rustpython-compiler-core",
 ]
@@ -1973,7 +2027,7 @@ dependencies = [
  "lexical-parse-float",
  "libc",
  "lock_api",
- "num-bigint",
+ "num-bigint 0.4.3",
  "num-complex",
  "num-traits",
  "once_cell",
@@ -1995,7 +2049,7 @@ dependencies = [
  "bstr",
  "itertools",
  "lz4_flex",
- "num-bigint",
+ "num-bigint 0.4.3",
  "num-complex",
  "serde",
  "static_assertions",
@@ -2013,7 +2067,7 @@ dependencies = [
  "lalrpop",
  "lalrpop-util",
  "log",
- "num-bigint",
+ "num-bigint 0.4.3",
  "num-traits",
  "phf 0.10.1",
  "phf_codegen 0.10.0",
@@ -2023,7 +2077,7 @@ dependencies = [
  "tiny-keccak",
  "unic-emoji-char",
  "unic-ucd-ident",
- "unicode_names2",
+ "unicode_names2 0.5.1",
 ]
 
 [[package]]
@@ -2315,7 +2369,7 @@ checksum = "76971977e6121664ec1b960d1313aacfa75642adc93b9d4d53b247bd4cb1747e"
 dependencies = [
  "dirs 2.0.2",
  "fnv",
- "nom",
+ "nom 5.1.2",
  "phf 0.8.0",
  "phf_codegen 0.8.0",
 ]
@@ -2485,12 +2539,24 @@ dependencies = [
  "tinyvec",
 ]
 
+[[package]]
+name = "unicode-xid"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
 
+[[package]]
+name = "unicode_names2"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dc6c5da0c8d7200f9488cc346bd30ba62bcd9f79ef937ea6573132e3d507df9"
+
 [[package]]
 name = "unicode_names2"
 version = "0.5.1"
@@ -2553,9 +2619,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55"
 dependencies = [
  "ctor",
- "version_check",
+ "version_check 0.9.4",
 ]
 
+[[package]]
+name = "version_check"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -2820,6 +2892,12 @@ dependencies = [
  "winapi-build",
 ]
 
+[[package]]
+name = "wtf8"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6b9309a86639c488a8eb2b5331cb5127cc9feb0a94a0db4b5d1ab5b84977956"
+
 [[package]]
 name = "yaml-rust"
 version = "0.4.5"

Cargo.toml

@@ -35,6 +35,7 @@ serde_json = { version = "1.0.83" }
 toml = { version = "0.5.9" }
 update-informer = { version = "0.5.0", default_features = false, features = ["pypi"], optional = true }
 walkdir = { version = "2.3.2" }
+python-parser = "0.2.0"
 
 [dev-dependencies]
 insta = { version = "1.19.1", features = ["yaml"] }

examples/parse_python.rs

@@ -0,0 +1,23 @@
+/// Print the AST for a given Python file.
+use std::path::PathBuf;
+
+use anyhow::Result;
+use clap::Parser;
+
+use ruff::fs;
+
+#[derive(Debug, Parser)]
+struct Cli {
+    #[arg(required = true)]
+    file: PathBuf,
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+
+    let contents = fs::read_file(&cli.file)?;
+    let ast = python_parser::file_input(python_parser::make_strspan(&contents))?.1;
+    println!("{:?}", ast);
+
+    Ok(())
+}

src/linter.rs

@@ -8,7 +8,6 @@ use rustpython_parser::{lexer, parser};
 
 use crate::ast::types::Range;
 use crate::autofix::fixer;
-use crate::autofix::fixer::fix_file;
 use crate::check_ast::check_ast;
 use crate::check_lines::check_lines;
 use crate::checks::{Check, CheckCode, CheckKind, LintSource};
@@ -101,33 +100,19 @@ pub fn lint_path(
     let contents = fs::read_file(path)?;
 
     // Tokenize once.
-    let tokens: Vec<LexResult> = tokenize(&contents);
+    // let tokens: Vec<LexResult> = tokenize(&contents);
 
     // Determine the noqa line for every line in the source.
-    let noqa_line_for = noqa::extract_noqa_line_for(&tokens);
+    // let noqa_line_for = noqa::extract_noqa_line_for(&tokens);
 
-    // Generate checks.
+    match python_parser::file_input(python_parser::make_strspan(&contents)) {
-    let mut checks = check_path(path, &contents, tokens, &noqa_line_for, settings, autofix)?;
+        Ok(x) => {}
+        Err(e) => return Err(anyhow::anyhow!("Err: failed")),
+    }
 
-    // Apply autofix.
+    // let _ = parser::parse_program_tokens(tokens, "<filename>")?;
-    if matches!(autofix, fixer::Mode::Apply) {
-        fix_file(&mut checks, &contents, path)?;
-    };
 
-    // Convert to messages.
+    Ok(vec![])
-    let messages: Vec<Message> = checks
-        .into_iter()
-        .map(|check| Message {
-            kind: check.kind,
-            fixed: check.fix.map(|fix| fix.applied).unwrap_or_default(),
-            location: check.location,
-            end_location: check.end_location,
-            filename: path.to_string_lossy().to_string(),
-        })
-        .collect();
-    cache::set(path, &metadata, settings, autofix, &messages, mode);
-
-    Ok(messages)
 }
 
 pub fn add_noqa_to_path(path: &Path, settings: &Settings) -> Result<usize> {