Use take-while to terminate on parse errors (#279)

src/checks.rs

@@ -256,7 +256,7 @@ impl CheckCode {
     pub fn lint_source(&self) -> &'static LintSource {
         match self {
             CheckCode::E501 | CheckCode::M001 => &LintSource::Lines,
-            CheckCode::E902 | CheckCode::E999 => &LintSource::FileSystem,
+            CheckCode::E902 => &LintSource::FileSystem,
             _ => &LintSource::AST,
         }
     }

src/linter.rs

@@ -15,19 +15,30 @@ use crate::noqa::add_noqa;
 use crate::settings::Settings;
 use crate::{cache, fs, noqa};
 
+/// Collect tokens up to and including the first error.
+fn tokenize(contents: &str) -> Vec<LexResult> {
+    let mut tokens: Vec<LexResult> = vec![];
+    for tok in lexer::make_tokenizer(contents) {
+        let is_err = tok.is_err();
+        tokens.push(tok);
+        if is_err {
+            break;
+        }
+    }
+    tokens
+}
+
 fn check_path(
     path: &Path,
     contents: &str,
     tokens: Vec<LexResult>,
+    noqa_line_for: &[usize],
     settings: &Settings,
     autofix: &fixer::Mode,
 ) -> Result<Vec<Check>> {
     // Aggregate all checks.
     let mut checks: Vec<Check> = vec![];
 
-    // Determine the noqa line for every line in the source.
-    let noqa_line_for = noqa::extract_noqa_line_for(&tokens);
-
     // Run the AST-based checks.
     if settings
         .select
@@ -50,7 +61,7 @@ fn check_path(
     }
 
     // Run the lines-based checks.
-    check_lines(&mut checks, contents, &noqa_line_for, settings, autofix);
+    check_lines(&mut checks, contents, noqa_line_for, settings, autofix);
 
     // Create path ignores.
     if !checks.is_empty() && !settings.per_file_ignores.is_empty() {
@@ -84,10 +95,13 @@ pub fn lint_path(
     let contents = fs::read_file(path)?;
 
     // Tokenize once.
-    let tokens: Vec<LexResult> = lexer::make_tokenizer(&contents).collect();
+    let tokens: Vec<LexResult> = tokenize(&contents);
+
+    // Determine the noqa line for every line in the source.
+    let noqa_line_for = noqa::extract_noqa_line_for(&tokens);
 
     // Generate checks.
-    let mut checks = check_path(path, &contents, tokens, settings, autofix)?;
+    let mut checks = check_path(path, &contents, tokens, &noqa_line_for, settings, autofix)?;
 
     // Apply autofix.
     if matches!(autofix, fixer::Mode::Apply) {
@@ -114,13 +128,20 @@ pub fn add_noqa_to_path(path: &Path, settings: &Settings) -> Result<usize> {
     let contents = fs::read_file(path)?;
 
     // Tokenize once.
-    let tokens: Vec<LexResult> = lexer::make_tokenizer(&contents).collect();
+    let tokens: Vec<LexResult> = tokenize(&contents);
 
     // Determine the noqa line for every line in the source.
     let noqa_line_for = noqa::extract_noqa_line_for(&tokens);
 
     // Generate checks.
-    let checks = check_path(path, &contents, tokens, settings, &fixer::Mode::None)?;
+    let checks = check_path(
+        path,
+        &contents,
+        tokens,
+        &noqa_line_for,
+        settings,
+        &fixer::Mode::None,
+    )?;
 
     add_noqa(&checks, &contents, &noqa_line_for, path)
 }
@@ -131,14 +152,14 @@ mod tests {
 
     use anyhow::Result;
     use regex::Regex;
-    use rustpython_parser::lexer;
     use rustpython_parser::lexer::LexResult;
 
     use crate::autofix::fixer;
     use crate::checks::{Check, CheckCode};
-    use crate::fs;
     use crate::linter;
+    use crate::linter::tokenize;
     use crate::settings;
+    use crate::{fs, noqa};
 
     fn check_path(
         path: &Path,
@@ -146,8 +167,9 @@ mod tests {
         autofix: &fixer::Mode,
     ) -> Result<Vec<Check>> {
         let contents = fs::read_file(path)?;
-        let tokens: Vec<LexResult> = lexer::make_tokenizer(&contents).collect();
-        linter::check_path(path, &contents, tokens, settings, autofix)
+        let tokens: Vec<LexResult> = tokenize(&contents);
+        let noqa_line_for = noqa::extract_noqa_line_for(&tokens);
+        linter::check_path(path, &contents, tokens, &noqa_line_for, settings, autofix)
     }
 
     #[test]
@@ -692,4 +714,16 @@ mod tests {
         insta::assert_yaml_snapshot!(checks);
         Ok(())
     }
+
+    #[test]
+    fn e999() -> Result<()> {
+        let mut checks = check_path(
+            Path::new("./resources/test/fixtures/E999.py"),
+            &settings::Settings::for_rule(CheckCode::E999),
+            &fixer::Mode::Generate,
+        )?;
+        checks.sort_by_key(|check| check.location);
+        insta::assert_yaml_snapshot!(checks);
+        Ok(())
+    }
 }

src/noqa.rs

@@ -1,13 +1,14 @@
 use std::cmp::{max, min};
+use std::collections::{BTreeMap, BTreeSet};
+use std::fs;
+use std::path::Path;
 
-use crate::checks::{Check, CheckCode};
 use anyhow::Result;
 use once_cell::sync::Lazy;
 use regex::Regex;
 use rustpython_parser::lexer::{LexResult, Tok};
-use std::collections::{BTreeMap, BTreeSet};
-use std::fs;
-use std::path::Path;
+
+use crate::checks::{Check, CheckCode};
 
 static NO_QA_REGEX: Lazy<Regex> = Lazy::new(|| {
     Regex::new(r"(?i)(?P<noqa>\s*# noqa(?::\s?(?P<codes>([A-Z]+[0-9]+(?:[,\s]+)?)+))?)")
@@ -160,12 +161,12 @@ pub fn add_noqa(
 
 #[cfg(test)]
 mod tests {
-    use crate::checks::{Check, CheckKind};
     use anyhow::Result;
     use rustpython_parser::ast::Location;
     use rustpython_parser::lexer;
     use rustpython_parser::lexer::LexResult;
 
+    use crate::checks::{Check, CheckKind};
     use crate::noqa::{add_noqa_inner, extract_noqa_line_for};
 
     #[test]

src/snapshots/ruff__linter__tests__e999.snap

@@ -0,0 +1,11 @@
+---
+source: src/linter.rs
+expression: checks
+---
+- kind:
+    SyntaxError: Got unexpected EOF
+  location:
+    row: 2
+    column: 1
+  fix: ~
+