Python
/
ruff
mirror of https://github.com/astral-sh/ruff
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Resolve hierarchical settings and Python files in a single filesystem pass (#1205)

This commit is contained in:
Charlie Marsh 2022-12-12 10:13:52 -05:00 committed by GitHub
parent 0adc9ed259
commit 73794fc299
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 98 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/commands.rs
View File

@ -1,13 +1,12 @@
use std::path::{Path, PathBuf};
use anyhow::{bail, Result};
use itertools::Itertools;
use serde::Serialize;
use walkdir::DirEntry;
use crate::checks::CheckCode;
use crate::cli::Overrides;
use crate::fs::iter_python_files;
use crate::resolver::{discover_settings, Resolver};
use crate::fs::collect_python_files;
use crate::settings::types::SerializationFormat;
use crate::{Configuration, Settings};
@ -24,23 +23,15 @@ pub fn show_settings(
/// Show the list of files to be checked based on current settings.
pub fn show_files(files: &[PathBuf], default: &Settings, overrides: &Overrides) {
// Discover the settings for the filesystem hierarchy.
let settings = discover_settings(files, overrides);
let resolver = Resolver {
default,
settings: &settings,
};
// Collect all files in the hierarchy.
let mut entries: Vec<DirEntry> = files
.iter()
.flat_map(|path| iter_python_files(path, &resolver))
.flatten()
.collect();
entries.sort_by(|a, b| a.path().cmp(b.path()));
let (paths, _resolver) = collect_python_files(files, overrides, default);
// Print the list of files.
for entry in entries {
for entry in paths
.iter()
.flatten()
.sorted_by(|a, b| a.path().cmp(b.path()))
{
println!("{}", entry.path().to_string_lossy());
}
}

94
src/fs.rs
View File

@ -5,13 +5,16 @@ use std::path::{Path, PathBuf};
use anyhow::{anyhow, Result};
use globset::GlobMatcher;
use log::debug;
use log::{debug, error};
use path_absolutize::{path_dedot, Absolutize};
use rustc_hash::FxHashSet;
use walkdir::{DirEntry, WalkDir};
use crate::checks::CheckCode;
use crate::cli::Overrides;
use crate::resolver;
use crate::resolver::Resolver;
use crate::settings::Settings;
/// Extract the absolute path and basename (as strings) from a Path.
fn extract_path_names(path: &Path) -> Result<(&str, &str)> {
@ -35,59 +38,71 @@ fn is_included(path: &Path) -> bool {
.map_or(false, |ext| ext == "py" || ext == "pyi")
}
/// Find all `pyproject.toml` files for a given `Path`. Both parents and
/// children will be included in the resulting `Vec`.
pub fn iter_pyproject_files(path: &Path) -> Vec<PathBuf> {
let mut paths = Vec::new();
/// Find all Python (`.py` and `.pyi` files) in a set of `Path`.
pub fn collect_python_files<'a>(
paths: &'a [PathBuf],
overrides: &'a Overrides,
default: &'a Settings,
) -> (Vec<Result<DirEntry, walkdir::Error>>, Resolver) {
let mut files = Vec::new();
let mut resolver = Resolver::default();
for path in paths {
let (files_in_path, file_resolver) = python_files_in_path(path, overrides, default);
files.extend(files_in_path);
resolver.merge(file_resolver);
}
(files, resolver)
}
/// Find all Python (`.py` and `.pyi` files) in a given `Path`.
fn python_files_in_path<'a>(
path: &'a Path,
overrides: &'a Overrides,
default: &'a Settings,
) -> (Vec<Result<DirEntry, walkdir::Error>>, Resolver) {
let path = normalize_path(path);
// Search for `pyproject.toml` files in all parent directories.
let path = normalize_path(path);
let mut resolver = Resolver::default();
for path in path.ancestors() {
if path.is_dir() {
let toml_path = path.join("pyproject.toml");
if toml_path.exists() {
paths.push(toml_path);
let pyproject = path.join("pyproject.toml");
if pyproject.is_file() {
match resolver::settings_for_path(&pyproject, overrides) {
Ok((root, settings)) => resolver.add(root, settings),
Err(err) => error!("Failed to read settings: {err}"),
}
}
}
}
// Search for `pyproject.toml` files in all child directories.
for path in WalkDir::new(path)
// Collect all Python files.
let files: Vec<Result<DirEntry, walkdir::Error>> = WalkDir::new(path)
.into_iter()
.filter_entry(|entry| {
entry.file_name().to_str().map_or(false, |file_name| {
entry.depth() == 0 || !file_name.starts_with('.')
})
})
.filter_map(std::result::Result::ok)
.filter(|entry| entry.path().ends_with("pyproject.toml"))
{
paths.push(path.into_path());
}
// Search for the `pyproject.toml` file in this directory, before we visit any
// of its contents.
if entry.file_type().is_dir() {
let pyproject = entry.path().join("pyproject.toml");
if pyproject.is_file() {
match resolver::settings_for_path(&pyproject, overrides) {
Ok((root, settings)) => resolver.add(root, settings),
Err(err) => error!("Failed to read settings: {err}"),
}
}
}
paths
}
/// Find all Python (`.py` and `.pyi` files) in a given `Path`.
pub fn iter_python_files<'a>(
path: &'a Path,
resolver: &'a Resolver<'a>,
) -> impl Iterator<Item = Result<DirEntry, walkdir::Error>> + 'a {
WalkDir::new(normalize_path(path))
.into_iter()
.filter_entry(move |entry| {
let path = entry.path();
let settings = resolver.resolve(path);
let exclude = &settings.exclude;
let extend_exclude = &settings.extend_exclude;
let settings = resolver.resolve(path).unwrap_or(default);
match extract_path_names(path) {
Ok((file_path, file_basename)) => {
if !exclude.is_empty() && is_excluded(file_path, file_basename, exclude) {
if !settings.exclude.is_empty()
&& is_excluded(file_path, file_basename, &settings.exclude)
{
debug!("Ignored path via `exclude`: {:?}", path);
false
} else if !extend_exclude.is_empty()
&& is_excluded(file_path, file_basename, extend_exclude)
} else if !settings.extend_exclude.is_empty()
&& is_excluded(file_path, file_basename, &settings.extend_exclude)
{
debug!("Ignored path via `extend-exclude`: {:?}", path);
false
@ -108,6 +123,9 @@ pub fn iter_python_files<'a>(
&& !(entry.file_type().is_symlink() && entry.path().is_dir())
})
})
.collect::<Vec<_>>();
(files, resolver)
}
/// Create tree set with codes matching the pattern/code pairs.

2
src/linter.rs
View File

@ -211,7 +211,7 @@ pub fn add_noqa_to_path(path: &Path, settings: &Settings) -> Result<usize> {
}
/// Apply autoformatting to the source code at the given `Path`.
pub fn autoformat_path(path: &Path) -> Result<()> {
pub fn autoformat_path(path: &Path, _settings: &Settings) -> Result<()> {
// Read the file from disk.
let contents = fs::read_file(path)?;

56
src/main.rs
View File

@ -20,13 +20,12 @@ use std::time::Instant;
use ::ruff::autofix::fixer;
use ::ruff::checks::{CheckCode, CheckKind};
use ::ruff::cli::{extract_log_level, Cli, Overrides};
use ::ruff::fs::iter_python_files;
use ::ruff::fs::collect_python_files;
use ::ruff::iterators::par_iter;
use ::ruff::linter::{add_noqa_to_path, autoformat_path, lint_path, lint_stdin, Diagnostics};
use ::ruff::logging::{set_up_logging, LogLevel};
use ::ruff::message::Message;
use ::ruff::printer::Printer;
use ::ruff::resolver::Resolver;
use ::ruff::settings::configuration::Configuration;
use ::ruff::settings::types::SerializationFormat;
use ::ruff::settings::{pyproject, Settings};
@ -40,9 +39,7 @@ use log::{debug, error};
use notify::{recommended_watcher, RecursiveMode, Watcher};
#[cfg(not(target_family = "wasm"))]
use rayon::prelude::*;
use ruff::resolver::discover_settings;
use rustpython_ast::Location;
use walkdir::DirEntry;
fn read_from_stdin() -> Result<String> {
let mut buffer = String::new();
@ -68,19 +65,9 @@ fn run_once(
cache: bool,
autofix: &fixer::Mode,
) -> Diagnostics {
// Discover the settings for the filesystem hierarchy.
let settings = discover_settings(files, overrides);
let resolver = Resolver {
default,
settings: &settings,
};
// Collect all the files to check.
let start = Instant::now();
let paths: Vec<Result<DirEntry, walkdir::Error>> = files
.iter()
.flat_map(|path| iter_python_files(path, &resolver))
.collect();
let (paths, resolver) = collect_python_files(files, overrides, default);
let duration = start.elapsed();
debug!("Identified files to lint in: {:?}", duration);
@ -90,7 +77,7 @@ fn run_once(
match entry {
Ok(entry) => {
let path = entry.path();
let settings = resolver.resolve(path);
let settings = resolver.resolve(path).unwrap_or(default);
lint_path(path, settings, &cache.into(), autofix)
.map_err(|e| (Some(path.to_owned()), e.to_string()))
}
@ -101,8 +88,8 @@ fn run_once(
)),
}
.unwrap_or_else(|(path, message)| {
if let Some(path) = path {
let settings = resolver.resolve(&path);
if let Some(path) = &path {
let settings = resolver.resolve(path).unwrap_or(default);
if settings.enabled.contains(&CheckCode::E902) {
Diagnostics::new(vec![Message {
kind: CheckKind::IOError(message),
@ -135,28 +122,18 @@ fn run_once(
}
fn add_noqa(files: &[PathBuf], default: &Settings, overrides: &Overrides) -> usize {
// Discover the settings for the filesystem hierarchy.
let settings = discover_settings(files, overrides);
let resolver = Resolver {
default,
settings: &settings,
};
// Collect all the files to check.
let start = Instant::now();
let paths: Vec<DirEntry> = files
.iter()
.flat_map(|path| iter_python_files(path, &resolver))
.flatten()
.collect();
let (paths, resolver) = collect_python_files(files, overrides, default);
let duration = start.elapsed();
debug!("Identified files to lint in: {:?}", duration);
let start = Instant::now();
let modifications: usize = par_iter(&paths)
.flatten()
.filter_map(|entry| {
let path = entry.path();
let settings = resolver.resolve(path);
let settings = resolver.resolve(path).unwrap_or(default);
match add_noqa_to_path(path, settings) {
Ok(count) => Some(count),
Err(e) => {
@ -174,28 +151,19 @@ fn add_noqa(files: &[PathBuf], default: &Settings, overrides: &Overrides) -> usi
}
fn autoformat(files: &[PathBuf], default: &Settings, overrides: &Overrides) -> usize {
// Discover the settings for the filesystem hierarchy.
let settings = discover_settings(files, overrides);
let resolver = Resolver {
default,
settings: &settings,
};
// Collect all the files to format.
let start = Instant::now();
let paths: Vec<DirEntry> = files
.iter()
.flat_map(|path| iter_python_files(path, &resolver))
.flatten()
.collect();
let (paths, resolver) = collect_python_files(files, overrides, default);
let duration = start.elapsed();
debug!("Identified files to lint in: {:?}", duration);
let start = Instant::now();
let modifications = par_iter(&paths)
.flatten()
.filter_map(|entry| {
let path = entry.path();
match autoformat_path(path) {
let settings = resolver.resolve(path).unwrap_or(default);
match autoformat_path(path, settings) {
Ok(()) => Some(()),
Err(e) => {
error!("Failed to autoformat {}: {e}", path.to_string_lossy());

56
src/resolver.rs
View File

@ -1,27 +1,36 @@
//! Discover and resolve `Settings` from the filesystem hierarchy.
use std::cmp::Reverse;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use anyhow::{anyhow, Result};
use log::error;
use crate::cli::Overrides;
use crate::fs::iter_pyproject_files;
use crate::settings::configuration::Configuration;
use crate::settings::{pyproject, Settings};
pub struct Resolver<'a> {
pub settings: &'a [(PathBuf, Settings)],
pub default: &'a Settings,
#[derive(Default)]
pub struct Resolver {
settings: BTreeMap<PathBuf, Settings>,
}
impl<'a> Resolver<'a> {
pub fn resolve(&'a self, path: &Path) -> &'a Settings {
self.settings
.iter()
.find(|(root, _)| path.starts_with(root))
.map_or(self.default, |(_, settings)| settings)
impl Resolver {
pub fn merge(&mut self, resolver: Resolver) {
self.settings.extend(resolver.settings);
}
pub fn add(&mut self, path: PathBuf, settings: Settings) {
self.settings.insert(path, settings);
}
pub fn resolve(&self, path: &Path) -> Option<&Settings> {
self.settings.iter().rev().find_map(|(root, settings)| {
if path.starts_with(root) {
Some(settings)
} else {
None
}
})
}
}
@ -37,26 +46,3 @@ pub fn settings_for_path(pyproject: &Path, overrides: &Overrides) -> Result<(Pat
let settings = Settings::from_configuration(configuration, Some(&project_root))?;
Ok((project_root, settings))
}
/// Discover all `Settings` objects within the relevant filesystem hierarchy.
pub fn discover_settings(files: &[PathBuf], overrides: &Overrides) -> Vec<(PathBuf, Settings)> {
// Collect all `pyproject.toml` files.
let mut pyprojects: Vec<PathBuf> = files
.iter()
.flat_map(|path| iter_pyproject_files(path))
.collect();
pyprojects.sort_unstable_by_key(|path| Reverse(path.to_string_lossy().len()));
pyprojects.dedup();
// Read every `pyproject.toml`.
pyprojects
.into_iter()
.filter_map(|pyproject| match settings_for_path(&pyproject, overrides) {
Ok((project_root, settings)) => Some((project_root, settings)),
Err(error) => {
error!("Failed to read settings: {error}");
None
}
})
.collect::<Vec<_>>()
}