From 1d5592d937601aee3762640ce10271e8cc8187bb Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Wed, 28 Sep 2022 22:06:35 -0400 Subject: [PATCH] Use take-while to terminate on parse errors (#279) --- src/checks.rs | 2 +- src/linter.rs | 58 ++++++++++++++++---- src/noqa.rs | 11 ++-- src/snapshots/ruff__linter__tests__e999.snap | 11 ++++ 4 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 src/snapshots/ruff__linter__tests__e999.snap diff --git a/src/checks.rs b/src/checks.rs index 2b05defbb1..aee8fa5fcb 100644 --- a/src/checks.rs +++ b/src/checks.rs @@ -256,7 +256,7 @@ impl CheckCode { pub fn lint_source(&self) -> &'static LintSource { match self { CheckCode::E501 | CheckCode::M001 => &LintSource::Lines, - CheckCode::E902 | CheckCode::E999 => &LintSource::FileSystem, + CheckCode::E902 => &LintSource::FileSystem, _ => &LintSource::AST, } } diff --git a/src/linter.rs b/src/linter.rs index d9377098a9..3c184e5a5b 100644 --- a/src/linter.rs +++ b/src/linter.rs @@ -15,19 +15,30 @@ use crate::noqa::add_noqa; use crate::settings::Settings; use crate::{cache, fs, noqa}; +/// Collect tokens up to and including the first error. +fn tokenize(contents: &str) -> Vec { + let mut tokens: Vec = vec![]; + for tok in lexer::make_tokenizer(contents) { + let is_err = tok.is_err(); + tokens.push(tok); + if is_err { + break; + } + } + tokens +} + fn check_path( path: &Path, contents: &str, tokens: Vec, + noqa_line_for: &[usize], settings: &Settings, autofix: &fixer::Mode, ) -> Result> { // Aggregate all checks. let mut checks: Vec = vec![]; - // Determine the noqa line for every line in the source. - let noqa_line_for = noqa::extract_noqa_line_for(&tokens); - // Run the AST-based checks. if settings .select @@ -50,7 +61,7 @@ fn check_path( } // Run the lines-based checks. - check_lines(&mut checks, contents, &noqa_line_for, settings, autofix); + check_lines(&mut checks, contents, noqa_line_for, settings, autofix); // Create path ignores. if !checks.is_empty() && !settings.per_file_ignores.is_empty() { @@ -84,10 +95,13 @@ pub fn lint_path( let contents = fs::read_file(path)?; // Tokenize once. - let tokens: Vec = lexer::make_tokenizer(&contents).collect(); + let tokens: Vec = tokenize(&contents); + + // Determine the noqa line for every line in the source. + let noqa_line_for = noqa::extract_noqa_line_for(&tokens); // Generate checks. - let mut checks = check_path(path, &contents, tokens, settings, autofix)?; + let mut checks = check_path(path, &contents, tokens, &noqa_line_for, settings, autofix)?; // Apply autofix. if matches!(autofix, fixer::Mode::Apply) { @@ -114,13 +128,20 @@ pub fn add_noqa_to_path(path: &Path, settings: &Settings) -> Result { let contents = fs::read_file(path)?; // Tokenize once. - let tokens: Vec = lexer::make_tokenizer(&contents).collect(); + let tokens: Vec = tokenize(&contents); // Determine the noqa line for every line in the source. let noqa_line_for = noqa::extract_noqa_line_for(&tokens); // Generate checks. - let checks = check_path(path, &contents, tokens, settings, &fixer::Mode::None)?; + let checks = check_path( + path, + &contents, + tokens, + &noqa_line_for, + settings, + &fixer::Mode::None, + )?; add_noqa(&checks, &contents, &noqa_line_for, path) } @@ -131,14 +152,14 @@ mod tests { use anyhow::Result; use regex::Regex; - use rustpython_parser::lexer; use rustpython_parser::lexer::LexResult; use crate::autofix::fixer; use crate::checks::{Check, CheckCode}; - use crate::fs; use crate::linter; + use crate::linter::tokenize; use crate::settings; + use crate::{fs, noqa}; fn check_path( path: &Path, @@ -146,8 +167,9 @@ mod tests { autofix: &fixer::Mode, ) -> Result> { let contents = fs::read_file(path)?; - let tokens: Vec = lexer::make_tokenizer(&contents).collect(); - linter::check_path(path, &contents, tokens, settings, autofix) + let tokens: Vec = tokenize(&contents); + let noqa_line_for = noqa::extract_noqa_line_for(&tokens); + linter::check_path(path, &contents, tokens, &noqa_line_for, settings, autofix) } #[test] @@ -692,4 +714,16 @@ mod tests { insta::assert_yaml_snapshot!(checks); Ok(()) } + + #[test] + fn e999() -> Result<()> { + let mut checks = check_path( + Path::new("./resources/test/fixtures/E999.py"), + &settings::Settings::for_rule(CheckCode::E999), + &fixer::Mode::Generate, + )?; + checks.sort_by_key(|check| check.location); + insta::assert_yaml_snapshot!(checks); + Ok(()) + } } diff --git a/src/noqa.rs b/src/noqa.rs index 6d1b446048..fce545966c 100644 --- a/src/noqa.rs +++ b/src/noqa.rs @@ -1,13 +1,14 @@ use std::cmp::{max, min}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::Path; -use crate::checks::{Check, CheckCode}; use anyhow::Result; use once_cell::sync::Lazy; use regex::Regex; use rustpython_parser::lexer::{LexResult, Tok}; -use std::collections::{BTreeMap, BTreeSet}; -use std::fs; -use std::path::Path; + +use crate::checks::{Check, CheckCode}; static NO_QA_REGEX: Lazy = Lazy::new(|| { Regex::new(r"(?i)(?P\s*# noqa(?::\s?(?P([A-Z]+[0-9]+(?:[,\s]+)?)+))?)") @@ -160,12 +161,12 @@ pub fn add_noqa( #[cfg(test)] mod tests { - use crate::checks::{Check, CheckKind}; use anyhow::Result; use rustpython_parser::ast::Location; use rustpython_parser::lexer; use rustpython_parser::lexer::LexResult; + use crate::checks::{Check, CheckKind}; use crate::noqa::{add_noqa_inner, extract_noqa_line_for}; #[test] diff --git a/src/snapshots/ruff__linter__tests__e999.snap b/src/snapshots/ruff__linter__tests__e999.snap new file mode 100644 index 0000000000..ffa105e96a --- /dev/null +++ b/src/snapshots/ruff__linter__tests__e999.snap @@ -0,0 +1,11 @@ +--- +source: src/linter.rs +expression: checks +--- +- kind: + SyntaxError: Got unexpected EOF + location: + row: 2 + column: 1 + fix: ~ +