Merge branch 'main' into bojan/unzip-async

This commit is contained in:
Bojan Serafimov 2024-01-09 16:23:26 -05:00
commit fc9b547d0b
56 changed files with 997 additions and 574 deletions

View File

@ -38,7 +38,7 @@ jobs:
rustup component add clippy
- uses: Swatinem/rust-cache@v2
- name: "Clippy"
run: cargo clippy --workspace --all-targets --all-features -- -D warnings
run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
cargo-test:
strategy:

67
Cargo.lock generated
View File

@ -557,22 +557,21 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
[[package]]
name = "cmake"
version = "0.1.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
dependencies = [
"cc",
]
[[package]]
name = "colorchoice"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
[[package]]
name = "colored"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8"
dependencies = [
"lazy_static",
"windows-sys 0.48.0",
]
[[package]]
name = "configparser"
version = "3.0.4"
@ -926,6 +925,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e"
dependencies = [
"crc32fast",
"libz-ng-sys",
"miniz_oxide",
]
@ -1659,6 +1659,16 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "libz-ng-sys"
version = "1.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81157dde2fd4ad2b45ea3a4bb47b8193b52a6346b678840d91d80d3c2cd166c5"
dependencies = [
"cmake",
"libc",
]
[[package]]
name = "libz-sys"
version = "1.1.14"
@ -2250,7 +2260,7 @@ dependencies = [
[[package]]
name = "pubgrub"
version = "0.2.1"
source = "git+https://github.com/zanieb/pubgrub?rev=78b8add6942766e5fb070bbda1de570e93d6399f#78b8add6942766e5fb070bbda1de570e93d6399f"
source = "git+https://github.com/zanieb/pubgrub?rev=866c0f2a87fee1e8abe804d40a2ee934de0973d7#866c0f2a87fee1e8abe804d40a2ee934de0973d7"
dependencies = [
"indexmap 2.1.0",
"log",
@ -2319,7 +2329,6 @@ dependencies = [
"bitflags 2.4.1",
"chrono",
"clap",
"colored",
"distribution-filename",
"distribution-types",
"fs-err",
@ -2333,6 +2342,7 @@ dependencies = [
"itertools 0.12.0",
"miette",
"mimalloc",
"owo-colors",
"pep440_rs 0.3.12",
"pep508_rs",
"platform-host",
@ -2363,6 +2373,7 @@ dependencies = [
"tokio",
"toml",
"tracing",
"tracing-durations-export",
"tracing-subscriber",
"tracing-tree",
"url",
@ -2417,7 +2428,6 @@ dependencies = [
"anyhow",
"chrono",
"clap",
"colored",
"distribution-filename",
"distribution-types",
"fs-err",
@ -2427,6 +2437,7 @@ dependencies = [
"install-wheel-rs",
"itertools 0.12.0",
"mimalloc",
"owo-colors",
"pep440_rs 0.3.12",
"pep508_rs",
"petgraph",
@ -2448,6 +2459,7 @@ dependencies = [
"tikv-jemallocator",
"tokio",
"tracing",
"tracing-durations-export",
"tracing-indicatif",
"tracing-subscriber",
"url",
@ -2631,12 +2643,12 @@ dependencies = [
name = "puffin-resolver"
version = "0.0.1"
dependencies = [
"anstream",
"anyhow",
"bitflags 2.4.1",
"cache-key",
"chrono",
"clap",
"colored",
"derivative",
"distribution-filename",
"distribution-types",
@ -2648,6 +2660,7 @@ dependencies = [
"install-wheel-rs",
"itertools 0.12.0",
"once_cell",
"owo-colors",
"pep440_rs 0.3.12",
"pep508_rs",
"petgraph",
@ -2697,8 +2710,8 @@ name = "puffin-warnings"
version = "0.0.1"
dependencies = [
"anstream",
"colored",
"once_cell",
"owo-colors",
"rustc-hash",
]
@ -3451,6 +3464,12 @@ dependencies = [
"is-terminal",
]
[[package]]
name = "svg"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d703a3635418d4e4d0e410009ddbfb65047ef9468b1d29afd3b057a5bc4c217"
[[package]]
name = "syn"
version = "1.0.109"
@ -3860,6 +3879,24 @@ dependencies = [
"valuable",
]
[[package]]
name = "tracing-durations-export"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d6bb8898f56f636911130c78cc528338a2bb0426bdfb5a8fb523f98fc8da46d"
dependencies = [
"anyhow",
"fs-err",
"itertools 0.12.0",
"once_cell",
"rustc-hash",
"serde",
"serde_json",
"svg",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "tracing-indicatif"
version = "0.3.6"

View File

@ -24,14 +24,16 @@ camino = { version = "1.1.6", features = ["serde1"] }
cargo-util = { version = "0.2.8" }
chrono = { version = "0.4.31" }
clap = { version = "4.4.13" }
colored = { version = "2.1.0" }
configparser = { version = "3.0.4" }
csv = { version = "1.3.0" }
data-encoding = { version = "2.5.0" }
derivative = { version = "2.2.0" }
directories = { version = "5.0.1" }
dirs = { version = "5.0.1" }
flate2 = { version = "1.0.28" }
# This tells flate2 (and all libraries that depend on it, including async_compression
# and async_zip) to use zlib-ng, which about 2x faster than the default flate2 backend
# at decompression. See https://github.com/rust-lang/flate2-rs#backends
flate2 = { version = "1.0.28", features = ["zlib-ng"], default-features = false }
fs-err = { version = "2.11.0" }
fs2 = { version = "0.4.3" }
futures = { version = "0.3.30" }
@ -49,10 +51,11 @@ mailparse = { version = "0.14.0" }
# For additional textwrap options: https://github.com/zkat/miette/pull/321, https://github.com/zkat/miette/pull/328
miette = { git = "https://github.com/zkat/miette.git", rev = "b0744462adbbfbb6d845f382db36be883c7f3c45" }
once_cell = { version = "1.19.0" }
owo-colors = { version = "3.5.0" }
petgraph = { version = "0.6.4" }
platform-info = { version = "2.0.2" }
plist = { version = "1.6.0" }
pubgrub = { git = "https://github.com/zanieb/pubgrub", rev = "78b8add6942766e5fb070bbda1de570e93d6399f" }
pubgrub = { git = "https://github.com/zanieb/pubgrub", rev = "866c0f2a87fee1e8abe804d40a2ee934de0973d7" }
pyo3 = { version = "0.20.2" }
pyo3-log = { version = "0.9.0"}
pyproject-toml = { version = "0.8.1" }
@ -81,6 +84,7 @@ tokio-util = { version = "0.7.10", features = ["compat"] }
toml = { version = "0.8.8" }
toml_edit = { version = "0.21.0" }
tracing = { version = "0.1.40" }
tracing-durations-export = { version = "0.1.0", features = ["plot"] }
tracing-indicatif = { version = "0.3.6" }
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
tracing-tree = { version = "0.3.0" }

View File

@ -141,23 +141,16 @@ pub struct CachedWheel {
impl CachedWheel {
/// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any`).
pub fn from_path(path: &Path) -> Result<Option<Self>> {
let Some(file_name) = path.file_name() else {
return Ok(None);
};
let Some(file_name) = file_name.to_str() else {
return Ok(None);
};
let Ok(filename) = WheelFilename::from_stem(file_name) else {
return Ok(None);
};
pub fn from_path(path: &Path) -> Option<Self> {
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;
if path.is_file() {
return Ok(None);
return None;
}
let path = path.to_path_buf();
Ok(Some(Self { filename, path }))
Some(Self { filename, path })
}
/// Convert a [`CachedWheel`] into a [`CachedRegistryDist`].

View File

@ -1,7 +1,7 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use pep440_rs::VersionSpecifiers;
use pep440_rs::{VersionSpecifiers, VersionSpecifiersParseError};
use pypi_types::{DistInfoMetadata, Hashes, Yanked};
/// Internal analog to [`pypi_types::File`].
@ -11,23 +11,26 @@ pub struct File {
pub filename: String,
pub hashes: Hashes,
pub requires_python: Option<VersionSpecifiers>,
pub size: Option<usize>,
pub size: Option<u64>,
pub upload_time: Option<DateTime<Utc>>,
pub url: String,
pub yanked: Option<Yanked>,
}
impl From<pypi_types::File> for File {
fn from(file: pypi_types::File) -> Self {
Self {
impl TryFrom<pypi_types::File> for File {
type Error = VersionSpecifiersParseError;
/// `TryFrom` instead of `From` to filter out files with invalid requires python version specifiers
fn try_from(file: pypi_types::File) -> Result<Self, Self::Error> {
Ok(Self {
dist_info_metadata: file.dist_info_metadata,
filename: file.filename,
hashes: file.hashes,
requires_python: file.requires_python,
requires_python: file.requires_python.transpose()?,
size: file.size,
upload_time: file.upload_time,
url: file.url,
yanked: file.yanked,
}
})
}
}

View File

@ -506,7 +506,7 @@ impl RemoteSource for File {
Ok(&self.filename)
}
fn size(&self) -> Option<usize> {
fn size(&self) -> Option<u64> {
self.size
}
}
@ -518,7 +518,7 @@ impl RemoteSource for Url {
.ok_or_else(|| Error::UrlFilename(self.clone()))
}
fn size(&self) -> Option<usize> {
fn size(&self) -> Option<u64> {
None
}
}
@ -528,7 +528,7 @@ impl RemoteSource for RegistryBuiltDist {
self.file.filename()
}
fn size(&self) -> Option<usize> {
fn size(&self) -> Option<u64> {
self.file.size()
}
}
@ -538,7 +538,7 @@ impl RemoteSource for RegistrySourceDist {
self.file.filename()
}
fn size(&self) -> Option<usize> {
fn size(&self) -> Option<u64> {
self.file.size()
}
}
@ -548,7 +548,7 @@ impl RemoteSource for DirectUrlBuiltDist {
self.url.filename()
}
fn size(&self) -> Option<usize> {
fn size(&self) -> Option<u64> {
self.url.size()
}
}
@ -558,7 +558,7 @@ impl RemoteSource for DirectUrlSourceDist {
self.url.filename()
}
fn size(&self) -> Option<usize> {
fn size(&self) -> Option<u64> {
self.url.size()
}
}
@ -572,7 +572,7 @@ impl RemoteSource for GitSourceDist {
})
}
fn size(&self) -> Option<usize> {
fn size(&self) -> Option<u64> {
self.url.size()
}
}
@ -582,7 +582,7 @@ impl RemoteSource for PathBuiltDist {
self.url.filename()
}
fn size(&self) -> Option<usize> {
fn size(&self) -> Option<u64> {
self.url.size()
}
}
@ -592,7 +592,7 @@ impl RemoteSource for PathSourceDist {
self.url.filename()
}
fn size(&self) -> Option<usize> {
fn size(&self) -> Option<u64> {
self.url.size()
}
}
@ -607,7 +607,7 @@ impl RemoteSource for SourceDist {
}
}
fn size(&self) -> Option<usize> {
fn size(&self) -> Option<u64> {
match self {
Self::Registry(dist) => dist.size(),
Self::DirectUrl(dist) => dist.size(),
@ -626,7 +626,7 @@ impl RemoteSource for BuiltDist {
}
}
fn size(&self) -> Option<usize> {
fn size(&self) -> Option<u64> {
match self {
Self::Registry(dist) => dist.size(),
Self::DirectUrl(dist) => dist.size(),
@ -643,7 +643,7 @@ impl RemoteSource for Dist {
}
}
fn size(&self) -> Option<usize> {
fn size(&self) -> Option<u64> {
match self {
Self::Built(dist) => dist.size(),
Self::Source(dist) => dist.size(),

View File

@ -53,7 +53,7 @@ pub trait RemoteSource {
fn filename(&self) -> Result<&str, Error>;
/// Return the size of the distribution, if known.
fn size(&self) -> Option<usize>;
fn size(&self) -> Option<u64>;
}
pub trait Identifier {

View File

@ -23,7 +23,7 @@ puffin-interpreter = { path = "../puffin-interpreter" }
anstream = { workspace = true }
camino = { workspace = true }
clap = { workspace = true }
clap = { workspace = true, features = ["derive"] }
fs-err = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }

View File

@ -6,7 +6,7 @@ use std::path::Path;
use configparser::ini::Ini;
use fs_err as fs;
use fs_err::File;
use tracing::{debug, info_span};
use tracing::{debug, instrument};
use pypi_types::DirectUrl;
@ -24,6 +24,7 @@ use crate::{read_record_file, Error, Script};
/// <https://packaging.python.org/en/latest/specifications/binary-distribution-format/#installing-a-wheel-distribution-1-0-py32-none-any-whl>
///
/// Wheel 1.0: <https://www.python.org/dev/peps/pep-0427/>
#[instrument(skip_all, fields(wheel = %wheel.as_ref().display()))]
pub fn install_wheel(
location: &InstallLocation<impl AsRef<Path>>,
wheel: impl AsRef<Path>,
@ -52,8 +53,6 @@ pub fn install_wheel(
let metadata = dist_info_metadata(&dist_info_prefix, &wheel)?;
let (name, _version) = parse_metadata(&dist_info_prefix, &metadata)?;
let _my_span = info_span!("install_wheel", name);
// We're going step by step though
// https://packaging.python.org/en/latest/specifications/binary-distribution-format/#installing-a-wheel-distribution-1-0-py32-none-any-whl
// > 1.a Parse distribution-1.0.dist-info/WHEEL.
@ -137,8 +136,9 @@ fn find_dist_info(path: impl AsRef<Path>) -> Result<String, Error> {
// Iterate over `path` to find the `.dist-info` directory. It should be at the top-level.
let Some(dist_info) = fs::read_dir(path.as_ref())?.find_map(|entry| {
let entry = entry.ok()?;
let path = entry.path();
if path.is_dir() {
let file_type = entry.file_type().ok()?;
if file_type.is_dir() {
let path = entry.path();
if path.extension().map_or(false, |ext| ext == "dist-info") {
Some(path)
} else {
@ -231,6 +231,7 @@ impl Default for LinkMode {
impl LinkMode {
/// Extract a wheel by linking all of its files into site packages.
#[instrument(skip_all)]
pub fn link_wheel_files(
self,
site_packages: impl AsRef<Path>,
@ -317,7 +318,9 @@ fn copy_wheel_files(
// Walk over the directory.
for entry in walkdir::WalkDir::new(&wheel) {
let entry = entry?;
let relative = entry.path().strip_prefix(&wheel).unwrap();
let path = entry.path();
let relative = path.strip_prefix(&wheel).unwrap();
let out_path = site_packages.as_ref().join(relative);
if entry.file_type().is_dir() {
@ -326,7 +329,7 @@ fn copy_wheel_files(
}
// Copy the file.
fs::copy(entry.path(), &out_path)?;
fs::copy(path, &out_path)?;
#[cfg(unix)]
{
@ -370,7 +373,9 @@ fn hardlink_wheel_files(
// Walk over the directory.
for entry in walkdir::WalkDir::new(&wheel) {
let entry = entry?;
let relative = entry.path().strip_prefix(&wheel).unwrap();
let path = entry.path();
let relative = path.strip_prefix(&wheel).unwrap();
let out_path = site_packages.as_ref().join(relative);
if entry.file_type().is_dir() {
@ -379,8 +384,8 @@ fn hardlink_wheel_files(
}
// The `RECORD` file is modified during installation, so we copy it instead of hard-linking.
if entry.path().ends_with("RECORD") {
fs::copy(entry.path(), &out_path)?;
if path.ends_with("RECORD") {
fs::copy(path, &out_path)?;
count += 1;
continue;
}
@ -390,33 +395,33 @@ fn hardlink_wheel_files(
Attempt::Initial => {
// Once https://github.com/rust-lang/rust/issues/86442 is stable, use that.
attempt = Attempt::Subsequent;
if let Err(err) = fs::hard_link(entry.path(), &out_path) {
if let Err(err) = fs::hard_link(path, &out_path) {
// If the file already exists, remove it and try again.
if err.kind() == std::io::ErrorKind::AlreadyExists {
fs::remove_file(&out_path)?;
if fs::hard_link(entry.path(), &out_path).is_err() {
fs::copy(entry.path(), &out_path)?;
if fs::hard_link(path, &out_path).is_err() {
fs::copy(path, &out_path)?;
attempt = Attempt::UseCopyFallback;
}
} else {
fs::copy(entry.path(), &out_path)?;
fs::copy(path, &out_path)?;
attempt = Attempt::UseCopyFallback;
}
}
}
Attempt::Subsequent => {
if let Err(err) = fs::hard_link(entry.path(), &out_path) {
if let Err(err) = fs::hard_link(path, &out_path) {
// If the file already exists, remove it and try again.
if err.kind() == std::io::ErrorKind::AlreadyExists {
fs::remove_file(&out_path)?;
fs::hard_link(entry.path(), &out_path)?;
fs::hard_link(path, &out_path)?;
} else {
return Err(err.into());
}
}
}
Attempt::UseCopyFallback => {
fs::copy(entry.path(), &out_path)?;
fs::copy(path, &out_path)?;
}
}

View File

@ -1,5 +1,5 @@
use std::collections::HashMap;
use std::ffi::OsString;
use std::io::{BufRead, BufReader, BufWriter, Cursor, Read, Seek, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, ExitStatus, Stdio};
@ -87,9 +87,8 @@ fn parse_scripts<R: Read + Seek>(
) -> Result<(Vec<Script>, Vec<Script>), Error> {
let entry_points_path = format!("{dist_info_dir}/entry_points.txt");
let entry_points_mapping = match archive.by_name(&entry_points_path) {
Ok(mut file) => {
let mut ini_text = String::new();
file.read_to_string(&mut ini_text)?;
Ok(file) => {
let ini_text = std::io::read_to_string(file)?;
Ini::new_cs()
.read(ini_text)
.map_err(|err| Error::InvalidWheel(format!("entry_points.txt is invalid: {err}")))?
@ -433,6 +432,7 @@ pub(crate) fn parse_wheel_version(wheel_text: &str) -> Result<(), Error> {
///
/// 2.f Compile any installed .py to .pyc. (Uninstallers should be smart enough to remove .pyc
/// even if it is not mentioned in RECORD.)
#[instrument(skip_all)]
fn bytecode_compile(
site_packages: &Path,
unpacked_paths: Vec<PathBuf>,
@ -446,7 +446,9 @@ fn bytecode_compile(
let py_source_paths: Vec<_> = unpacked_paths
.into_iter()
.filter(|path| {
site_packages.join(path).is_file() && path.extension() == Some(&OsString::from("py"))
path.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("py"))
&& site_packages.join(path).is_file()
})
.collect();
@ -655,16 +657,18 @@ fn install_script(
file: &DirEntry,
location: &InstallLocation<impl AsRef<Path>>,
) -> Result<(), Error> {
let path = file.path();
if !path.is_file() {
if !file.file_type()?.is_file() {
return Err(Error::InvalidWheel(format!(
"Wheel contains entry in scripts directory that is not a file: {}",
path.display()
file.path().display()
)));
}
let target_path = bin_rel().join(file.file_name());
let path = file.path();
let mut script = File::open(&path)?;
// https://sphinx-locales.github.io/peps/pep-0427/#recommended-installer-features
// > In wheel, scripts are packaged in {distribution}-{version}.data/scripts/.
// > If the first line of a file in scripts/ starts with exactly b'#!python',
@ -724,6 +728,7 @@ fn install_script(
/// Move the files from the .data directory to the right location in the venv
#[allow(clippy::too_many_arguments)]
#[instrument(skip_all)]
pub(crate) fn install_data(
venv_root: &Path,
site_packages: &Path,
@ -734,15 +739,17 @@ pub(crate) fn install_data(
gui_scripts: &[Script],
record: &mut [RecordEntry],
) -> Result<(), Error> {
for data_entry in fs::read_dir(data_dir)? {
let data_entry = data_entry?;
match data_entry.file_name().as_os_str().to_str() {
for entry in fs::read_dir(data_dir)? {
let entry = entry?;
let path = entry.path();
match path.file_name().and_then(|name| name.to_str()) {
Some("data") => {
// Move the content of the folder to the root of the venv
move_folder_recorded(&data_entry.path(), venv_root, site_packages, record)?;
move_folder_recorded(&path, venv_root, site_packages, record)?;
}
Some("scripts") => {
for file in fs::read_dir(data_entry.path())? {
for file in fs::read_dir(path)? {
let file = file?;
// Couldn't find any docs for this, took it directly from
@ -774,17 +781,17 @@ pub(crate) fn install_data(
location.python_version().1
))
.join(dist_name);
move_folder_recorded(&data_entry.path(), &target_path, site_packages, record)?;
move_folder_recorded(&path, &target_path, site_packages, record)?;
}
Some("purelib" | "platlib") => {
// purelib and platlib locations are not relevant when using venvs
// https://stackoverflow.com/a/27882460/3549270
move_folder_recorded(&data_entry.path(), site_packages, site_packages, record)?;
move_folder_recorded(&path, site_packages, site_packages, record)?;
}
_ => {
return Err(Error::InvalidWheel(format!(
"Unknown wheel data type: {:?}",
data_entry.file_name()
entry.file_name()
)));
}
}
@ -961,11 +968,10 @@ pub fn install_wheel(
// > 1.a Parse distribution-1.0.dist-info/WHEEL.
// > 1.b Check that installer is compatible with Wheel-Version. Warn if minor version is greater, abort if major version is greater.
let wheel_file_path = format!("{dist_info_prefix}.dist-info/WHEEL");
let mut wheel_text = String::new();
archive
let wheel_file = archive
.by_name(&wheel_file_path)
.map_err(|err| Error::Zip(wheel_file_path, err))?
.read_to_string(&mut wheel_text)?;
.map_err(|err| Error::Zip(wheel_file_path, err))?;
let wheel_text = io::read_to_string(wheel_file)?;
parse_wheel_version(&wheel_text)?;
// > 1.c If Root-Is-Purelib == true, unpack archive into purelib (site-packages).
// > 1.d Else unpack archive into platlib (site-packages).

View File

@ -284,39 +284,40 @@ impl SourceBuild {
};
// Check if we have a PEP 517 build backend.
let pep517_backend = if source_tree.join("pyproject.toml").is_file() {
let pyproject_toml: PyProjectToml =
toml::from_str(&fs::read_to_string(source_tree.join("pyproject.toml"))?)
.map_err(Error::InvalidPyprojectToml)?;
if let Some(build_system) = pyproject_toml.build_system {
Some(Pep517Backend {
// If `build-backend` is missing, inject the legacy setuptools backend, but
// retain the `requires`, to match `pip` and `build`. Note that while PEP 517
// says that in this case we "should revert to the legacy behaviour of running
// `setup.py` (either directly, or by implicitly invoking the
// `setuptools.build_meta:__legacy__` backend)", we found that in practice, only
// the legacy setuptools backend is allowed. See also:
// https://github.com/pypa/build/blob/de5b44b0c28c598524832dff685a98d5a5148c44/src/build/__init__.py#L114-L118
backend: build_system
.build_backend
.unwrap_or_else(|| "setuptools.build_meta:__legacy__".to_string()),
backend_path: build_system.backend_path,
requirements: build_system.requires,
})
} else {
// If a `pyproject.toml` is present, but `[build-system]` is missing, proceed with
// a PEP 517 build using the default backend, to match `pip` and `build`.
Some(Pep517Backend {
backend: "setuptools.build_meta:__legacy__".to_string(),
backend_path: None,
requirements: vec![
Requirement::from_str("wheel").unwrap(),
Requirement::from_str("setuptools >= 40.8.0").unwrap(),
],
})
let pep517_backend = match fs::read_to_string(source_tree.join("pyproject.toml")) {
Ok(toml) => {
let pyproject_toml: PyProjectToml =
toml::from_str(&toml).map_err(Error::InvalidPyprojectToml)?;
if let Some(build_system) = pyproject_toml.build_system {
Some(Pep517Backend {
// If `build-backend` is missing, inject the legacy setuptools backend, but
// retain the `requires`, to match `pip` and `build`. Note that while PEP 517
// says that in this case we "should revert to the legacy behaviour of running
// `setup.py` (either directly, or by implicitly invoking the
// `setuptools.build_meta:__legacy__` backend)", we found that in practice, only
// the legacy setuptools backend is allowed. See also:
// https://github.com/pypa/build/blob/de5b44b0c28c598524832dff685a98d5a5148c44/src/build/__init__.py#L114-L118
backend: build_system
.build_backend
.unwrap_or_else(|| "setuptools.build_meta:__legacy__".to_string()),
backend_path: build_system.backend_path,
requirements: build_system.requires,
})
} else {
// If a `pyproject.toml` is present, but `[build-system]` is missing, proceed with
// a PEP 517 build using the default backend, to match `pip` and `build`.
Some(Pep517Backend {
backend: "setuptools.build_meta:__legacy__".to_string(),
backend_path: None,
requirements: vec![
Requirement::from_str("wheel").unwrap(),
Requirement::from_str("setuptools >= 40.8.0").unwrap(),
],
})
}
}
} else {
None
Err(err) if err.kind() == io::ErrorKind::NotFound => None,
Err(err) => return Err(err.into()),
};
let venv = gourgeist::create_venv(&temp_dir.path().join(".venv"), interpreter.clone())?;

View File

@ -8,6 +8,7 @@ documentation = { workspace = true }
repository = { workspace = true }
authors = { workspace = true }
license = { workspace = true }
default-run = "puffin"
[lints]
workspace = true
@ -45,12 +46,12 @@ anyhow = { workspace = true }
bitflags = { workspace = true }
chrono = { workspace = true }
clap = { workspace = true, features = ["derive"] }
colored = { workspace = true }
fs-err = { workspace = true, features = ["tokio"] }
futures = { workspace = true }
indicatif = { workspace = true }
itertools = { workspace = true }
miette = { workspace = true, features = ["fancy"] }
owo-colors = { workspace = true }
pubgrub = { workspace = true }
pyproject-toml = { workspace = true }
rustc-hash = { workspace = true }
@ -60,6 +61,7 @@ thiserror = { workspace = true }
tokio = { workspace = true }
toml = { workspace = true }
tracing = { workspace = true }
tracing-durations-export = { workspace = true, features = ["plot"], optional = true }
tracing-subscriber = { workspace = true }
tracing-tree = { workspace = true }
url = { workspace = true }

View File

@ -1,8 +1,8 @@
use std::fmt::Write;
use anyhow::{Context, Result};
use colored::Colorize;
use fs_err as fs;
use owo_colors::OwoColorize;
use puffin_cache::Cache;
use puffin_normalize::PackageName;
@ -20,7 +20,7 @@ pub(crate) fn clean(
writeln!(
printer,
"No cache found at: {}",
format!("{}", cache.root().display()).cyan()
cache.root().display().cyan()
)?;
return Ok(ExitStatus::Success);
}
@ -29,7 +29,7 @@ pub(crate) fn clean(
writeln!(
printer,
"Clearing cache at: {}",
format!("{}", cache.root().display()).cyan()
cache.root().display().cyan()
)?;
fs::remove_dir_all(cache.root())
.with_context(|| format!("Failed to clear cache at: {}", cache.root().display()))?;
@ -37,20 +37,12 @@ pub(crate) fn clean(
for package in packages {
let count = cache.purge(package)?;
match count {
0 => writeln!(
printer,
"No entries found for package: {}",
format!("{package}").cyan()
)?,
1 => writeln!(
printer,
"Cleared 1 entry for package: {}",
format!("{package}").cyan()
)?,
0 => writeln!(printer, "No entries found for package: {}", package.cyan())?,
1 => writeln!(printer, "Cleared 1 entry for package: {}", package.cyan())?,
count => writeln!(
printer,
"Cleared {count} entries for package: {}",
format!("{package}").cyan()
package.cyan()
)?,
}
}

View File

@ -1,8 +1,8 @@
use std::fmt::Write;
use anyhow::Result;
use colored::Colorize;
use itertools::Itertools;
use owo_colors::OwoColorize;
use tracing::debug;
use distribution_types::Name;

View File

@ -8,8 +8,8 @@ use std::str::FromStr;
use anstream::AutoStream;
use anyhow::{anyhow, Context, Result};
use chrono::{DateTime, Utc};
use colored::Colorize;
use itertools::Itertools;
use owo_colors::OwoColorize;
use tempfile::tempdir_in;
use tracing::debug;

View File

@ -2,10 +2,11 @@ use std::env;
use std::fmt::Write;
use std::path::Path;
use anstream::eprint;
use anyhow::{anyhow, Context, Result};
use chrono::{DateTime, Utc};
use colored::Colorize;
use itertools::Itertools;
use owo_colors::OwoColorize;
use tempfile::tempdir_in;
use tracing::debug;

View File

@ -1,8 +1,8 @@
use std::fmt::Write;
use anyhow::{Context, Result};
use colored::Colorize;
use itertools::Itertools;
use owo_colors::OwoColorize;
use tracing::debug;
use distribution_types::{IndexUrls, InstalledMetadata, LocalDist, LocalEditable, Name};

View File

@ -1,7 +1,7 @@
use std::fmt::Write;
use anyhow::Result;
use colored::Colorize;
use owo_colors::OwoColorize;
use tracing::debug;
use distribution_types::{InstalledMetadata, Name};

View File

@ -1,8 +1,8 @@
use std::sync::{Arc, Mutex};
use std::time::Duration;
use colored::Colorize;
use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
use owo_colors::OwoColorize;
use url::Url;
use distribution_types::{

View File

@ -2,9 +2,9 @@ use std::fmt::Write;
use std::path::{Path, PathBuf};
use anyhow::Result;
use colored::Colorize;
use fs_err as fs;
use miette::{Diagnostic, IntoDiagnostic};
use owo_colors::OwoColorize;
use thiserror::Error;
use platform_host::Platform;
@ -84,14 +84,14 @@ fn venv_impl(
printer,
"Using Python {} at {}",
interpreter.version(),
format!("{}", interpreter.sys_executable().display()).cyan()
interpreter.sys_executable().display().cyan()
)
.into_diagnostic()?;
writeln!(
printer,
"Creating virtual environment at: {}",
format!("{}", path.display()).cyan()
path.display().cyan()
)
.into_diagnostic()?;

View File

@ -1,7 +1,11 @@
use tracing::level_filters::LevelFilter;
#[cfg(feature = "tracing-durations-export")]
use tracing_durations_export::{
plot::PlotConfig, DurationsLayer, DurationsLayerBuilder, DurationsLayerDropGuard,
};
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::EnvFilter;
use tracing_subscriber::{EnvFilter, Layer, Registry};
use tracing_tree::time::Uptime;
use tracing_tree::HierarchicalLayer;
@ -20,7 +24,7 @@ pub(crate) enum Level {
/// The [`Level`] is used to dictate the default filters (which can be overridden by the `RUST_LOG`
/// environment variable) along with the formatting of the output. For example, [`Level::Verbose`]
/// includes targets and timestamps, along with all `puffin=debug` messages by default.
pub(crate) fn setup_logging(level: Level) {
pub(crate) fn setup_logging(level: Level, duration: impl Layer<Registry> + Send + Sync) {
match level {
Level::Default => {
// Show nothing, but allow `RUST_LOG` to override.
@ -30,6 +34,7 @@ pub(crate) fn setup_logging(level: Level) {
// Regardless of the tracing level, show messages without any adornment.
tracing_subscriber::registry()
.with(duration)
.with(filter)
.with(
tracing_subscriber::fmt::layer()
@ -47,6 +52,7 @@ pub(crate) fn setup_logging(level: Level) {
// Regardless of the tracing level, include the uptime and target for each message.
tracing_subscriber::registry()
.with(duration)
.with(filter)
.with(
HierarchicalLayer::default()
@ -58,3 +64,37 @@ pub(crate) fn setup_logging(level: Level) {
}
}
}
/// Setup the `TRACING_DURATIONS_FILE` environment variable to enable tracing durations.
#[cfg(feature = "tracing-durations-export")]
pub(crate) fn setup_duration() -> (
Option<DurationsLayer<Registry>>,
Option<DurationsLayerDropGuard>,
) {
if let Ok(location) = std::env::var("TRACING_DURATIONS_FILE") {
let location = std::path::PathBuf::from(location);
if let Some(parent) = location.parent() {
fs_err::create_dir_all(parent)
.expect("Failed to create parent of TRACING_DURATIONS_FILE");
}
let plot_config = PlotConfig {
multi_lane: true,
min_length: Some(std::time::Duration::from_secs_f32(0.002)),
remove: Some(
["get_cached_with_callback".to_string()]
.into_iter()
.collect(),
),
..PlotConfig::default()
};
let (layer, guard) = DurationsLayerBuilder::default()
.durations_file(&location)
.plot_file(location.with_extension("svg"))
.plot_config(plot_config)
.build()
.expect("Couldn't create TRACING_DURATIONS_FILE files");
(Some(layer), Some(guard))
} else {
(None, None)
}
}

View File

@ -6,7 +6,7 @@ use anstream::eprintln;
use anyhow::Result;
use chrono::{DateTime, Days, NaiveDate, NaiveTime, Utc};
use clap::{Args, Parser, Subcommand};
use colored::Colorize;
use owo_colors::OwoColorize;
use distribution_types::{IndexUrl, IndexUrls};
use puffin_cache::{Cache, CacheArgs};
@ -415,11 +415,18 @@ async fn inner() -> Result<ExitStatus> {
let cli = Cli::parse();
// Configure the `tracing` crate, which controls internal logging.
logging::setup_logging(if cli.verbose {
logging::Level::Verbose
} else {
logging::Level::Default
});
#[cfg(feature = "tracing-durations-export")]
let (duration_layer, _duration_guard) = logging::setup_duration();
#[cfg(not(feature = "tracing-durations-export"))]
let duration_layer = None::<tracing_subscriber::layer::Identity>;
logging::setup_logging(
if cli.verbose {
logging::Level::Verbose
} else {
logging::Level::Default
},
duration_layer,
);
// Configure the `Printer`, which controls user-facing output in the CLI.
let printer = if cli.quiet {

View File

@ -670,8 +670,8 @@ fn compile_python_37() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of Python available matching >=3.8 and
black==23.10.1 depends on Python>=3.8, black==23.10.1 is forbidden.
Because there are no versions of Python>=3.8 and black==23.10.1 depends
on Python>=3.8, black==23.10.1 is forbidden.
And because root depends on black==23.10.1, version solving failed.
"###);
});
@ -1405,8 +1405,8 @@ fn conflicting_direct_url_dependency() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of werkzeug available matching ==3.0.0 and
root depends on werkzeug==3.0.0, version solving failed.
Because there is no version of werkzeug==3.0.0 and root depends on
werkzeug==3.0.0, version solving failed.
"###);
});
@ -1555,8 +1555,8 @@ fn conflicting_transitive_url_dependency() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because flask==3.0.0 depends on werkzeug>=3.0.0 and there is no version
of werkzeug available matching >=3.0.0, flask==3.0.0 is forbidden.
Because flask==3.0.0 depends on werkzeug>=3.0.0 and there are no
versions of werkzeug>=3.0.0, flask==3.0.0 is forbidden.
And because root depends on flask==3.0.0, version solving failed.
"###);
});
@ -1899,8 +1899,8 @@ dependencies = ["django==300.1.4"]
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of django available matching ==300.1.4 and
my-project depends on django==300.1.4, version solving failed.
Because there is no version of django==300.1.4 and my-project depends on
django==300.1.4, version solving failed.
"###);
});
@ -2225,8 +2225,8 @@ fn compile_yanked_version_indirect() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of attrs available matching >20.3.0, <21.2.0
and root depends on attrs>20.3.0, <21.2.0, version solving failed.
Because there are no versions of attrs>20.3.0, <21.2.0 and root depends
on attrs>20.3.0, <21.2.0, version solving failed.
"###);
});

View File

@ -77,9 +77,8 @@ fn no_solution() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of flask available matching >3.0.0 and
flask==3.0.0 depends on werkzeug>=3.0.0, flask>=3.0.0 depends on
werkzeug>=3.0.0.
Because there are no versions of flask>3.0.0 and flask==3.0.0 depends on
werkzeug>=3.0.0, flask>=3.0.0 depends on werkzeug>=3.0.0.
And because root depends on flask>=3.0.0 and root depends on
werkzeug<1.0.0, version solving failed.
"###);
@ -644,7 +643,7 @@ fn install_editable_and_registry() -> Result<()> {
Resolved 1 package in [TIME]
Installed 1 package in [TIME]
- black==23.11.0
+ black==0.1.0 (from file://[WORKSPACE_DIR]/scripts/editable-installs/black_editable/)
+ black==0.1.0+editable (from file://[WORKSPACE_DIR]/scripts/editable-installs/black_editable/)
"###);
});
@ -694,7 +693,7 @@ fn install_editable_and_registry() -> Result<()> {
Resolved 6 packages in [TIME]
Downloaded 1 package in [TIME]
Installed 1 package in [TIME]
- black==0.1.0 (from file://[WORKSPACE_DIR]/scripts/editable-installs/black_editable/)
- black==0.1.0+editable (from file://[WORKSPACE_DIR]/scripts/editable-installs/black_editable/)
+ black==23.10.0
"###);
});

View File

@ -82,7 +82,7 @@ fn excluded_only_version() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of a available matching <1.0.0 | >1.0.0 and root depends on a<1.0.0 | >1.0.0, version solving failed.
Because there are no versions of a<1.0.0 | >1.0.0 and root depends on a<1.0.0 | >1.0.0, version solving failed.
"###);
});
@ -150,7 +150,7 @@ fn excluded_only_compatible_version() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of a available matching <1.0.0 | >1.0.0, <2.0.0 | >2.0.0, <3.0.0 | >3.0.0 and a==1.0.0 depends on b==1.0.0, a<2.0.0 depends on b==1.0.0.
Because there are no versions of a<1.0.0 | >1.0.0, <2.0.0 | >2.0.0, <3.0.0 | >3.0.0 and a==1.0.0 depends on b==1.0.0, a<2.0.0 depends on b==1.0.0.
And because a==3.0.0 depends on b==3.0.0, a<2.0.0 | >2.0.0 depends on b<=1.0.0 | >=3.0.0.
And because root depends on b>=2.0.0, <3.0.0 and root depends on a<2.0.0 | >2.0.0, version solving failed.
"###);
@ -258,11 +258,11 @@ fn dependency_excludes_range_of_compatible_versions() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of a available matching <1.0.0 | >1.0.0, <2.0.0 | >3.0.0 and a==1.0.0 depends on b==1.0.0, a<2.0.0 depends on b==1.0.0. (1)
Because there are no versions of a<1.0.0 | >1.0.0, <2.0.0 | >3.0.0 and a==1.0.0 depends on b==1.0.0, a<2.0.0 depends on b==1.0.0. (1)
Because there is no version of c available matching <1.0.0 | >1.0.0, <2.0.0 | >2.0.0 and c==1.0.0 depends on a<2.0.0, c<2.0.0 depends on a<2.0.0.
Because there are no versions of c<1.0.0 | >1.0.0, <2.0.0 | >2.0.0 and c==1.0.0 depends on a<2.0.0, c<2.0.0 depends on a<2.0.0.
And because c==2.0.0 depends on a>=3.0.0, c depends on a<2.0.0 | >=3.0.0.
And because a<2.0.0 depends on b==1.0.0 (1), a Not ( ==3.0.0 ), c *, b Not ( ==1.0.0 ) are incompatible.
And because a<2.0.0 depends on b==1.0.0 (1), a!=3.0.0, c*, b!=1.0.0 are incompatible.
And because a==3.0.0 depends on b==3.0.0, c depends on b<=1.0.0 | >=3.0.0.
And because root depends on c and root depends on b>=2.0.0, <3.0.0, version solving failed.
"###);
@ -386,10 +386,10 @@ fn dependency_excludes_non_contiguous_range_of_compatible_versions() -> Result<(
----- stderr -----
× No solution found when resolving dependencies:
Because a==1.0.0 depends on b==1.0.0 and there is no version of a available matching <1.0.0 | >1.0.0, <2.0.0 | >3.0.0, a<2.0.0 depends on b==1.0.0.
Because a==1.0.0 depends on b==1.0.0 and there are no versions of a<1.0.0 | >1.0.0, <2.0.0 | >3.0.0, a<2.0.0 depends on b==1.0.0.
And because a==3.0.0 depends on b==3.0.0, a<2.0.0 | >=3.0.0 depends on b<=1.0.0 | >=3.0.0. (1)
Because there is no version of c available matching <1.0.0 | >1.0.0, <2.0.0 | >2.0.0 and c==1.0.0 depends on a<2.0.0, c<2.0.0 depends on a<2.0.0.
Because there are no versions of c<1.0.0 | >1.0.0, <2.0.0 | >2.0.0 and c==1.0.0 depends on a<2.0.0, c<2.0.0 depends on a<2.0.0.
And because c==2.0.0 depends on a>=3.0.0, c depends on a<2.0.0 | >=3.0.0.
And because a<2.0.0 | >=3.0.0 depends on b<=1.0.0 | >=3.0.0 (1), c depends on b<=1.0.0 | >=3.0.0.
And because root depends on b>=2.0.0, <3.0.0 and root depends on c, version solving failed.
@ -521,7 +521,9 @@ fn requires_package_only_prereleases_in_range() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of a available matching >0.1.0 and root depends on a>0.1.0, version solving failed.
Because there are no versions of a>0.1.0 and root depends on a>0.1.0, version solving failed.
hint: Pre-releases are available for a in the requested range (e.g., 1.0.0a1), but pre-releases weren't enabled (try: `--prerelease=allow`)
"###);
});
@ -1114,8 +1116,10 @@ fn requires_transitive_package_only_prereleases_in_range() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of b available matching >0.1 and a==0.1.0 depends on b>0.1, a==0.1.0 is forbidden.
And because there is no version of a available matching <0.1.0 | >0.1.0 and root depends on a, version solving failed.
Because there are no versions of b>0.1 and a==0.1.0 depends on b>0.1, a==0.1.0 is forbidden.
And because there are no versions of a<0.1.0 | >0.1.0 and root depends on a, version solving failed.
hint: Pre-releases are available for b in the requested range (e.g., 1.0.0a1), but pre-releases weren't enabled (try: `--prerelease=allow`)
"###);
});
@ -1267,8 +1271,8 @@ fn requires_transitive_prerelease_and_stable_dependency() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of c available matching ==2.0.0b1 and a==1.0.0 depends on c==2.0.0b1, a==1.0.0 is forbidden.
And because there is no version of a available matching <1.0.0 | >1.0.0 and root depends on a, version solving failed.
Because there is no version of c==2.0.0b1 and a==1.0.0 depends on c==2.0.0b1, a==1.0.0 is forbidden.
And because there are no versions of a<1.0.0 | >1.0.0 and root depends on a, version solving failed.
hint: c was requested with a pre-release marker (e.g., ==2.0.0b1), but pre-releases weren't enabled (try: `--prerelease=allow`)
"###);
@ -1464,9 +1468,9 @@ fn requires_transitive_prerelease_and_stable_dependency_many_versions() -> Resul
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of b available matching <1.0.0 | >1.0.0 and b==1.0.0 depends on c, b depends on c.
And because there is no version of c available matching >=2.0.0b1, b depends on c<2.0.0b1.
And because a==1.0.0 depends on c>=2.0.0b1 and there is no version of a available matching <1.0.0 | >1.0.0, b *, a * are incompatible.
Because there are no versions of b<1.0.0 | >1.0.0 and b==1.0.0 depends on c, b depends on c.
And because there are no versions of c>=2.0.0b1, b depends on c<2.0.0b1.
And because a==1.0.0 depends on c>=2.0.0b1 and there are no versions of a<1.0.0 | >1.0.0, b*, a* are incompatible.
And because root depends on b and root depends on a, version solving failed.
hint: c was requested with a pre-release marker (e.g., >=2.0.0b1), but pre-releases weren't enabled (try: `--prerelease=allow`)
@ -1563,8 +1567,8 @@ fn requires_transitive_prerelease_and_stable_dependency_many_versions_holes() ->
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of c available matching >1.0.0, <2.0.0a5 | >2.0.0a7, <2.0.0b1 | >2.0.0b1, <2.0.0b5 and a==1.0.0 depends on c>1.0.0, <2.0.0a5 | >2.0.0a7, <2.0.0b1 | >2.0.0b1, <2.0.0b5, a==1.0.0 is forbidden.
And because there is no version of a available matching <1.0.0 | >1.0.0 and root depends on a, version solving failed.
Because there are no versions of c>1.0.0, <2.0.0a5 | >2.0.0a7, <2.0.0b1 | >2.0.0b1, <2.0.0b5 and a==1.0.0 depends on c>1.0.0, <2.0.0a5 | >2.0.0a7, <2.0.0b1 | >2.0.0b1, <2.0.0b5, a==1.0.0 is forbidden.
And because there are no versions of a<1.0.0 | >1.0.0 and root depends on a, version solving failed.
hint: c was requested with a pre-release marker (e.g., >1.0.0, <2.0.0a5 | >2.0.0a7, <2.0.0b1 | >2.0.0b1, <2.0.0b5), but pre-releases weren't enabled (try: `--prerelease=allow`)
"###);
@ -1677,7 +1681,7 @@ fn requires_exact_version_does_not_exist() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of a available matching ==2.0.0 and root depends on a==2.0.0, version solving failed.
Because there is no version of a==2.0.0 and root depends on a==2.0.0, version solving failed.
"###);
});
@ -1733,7 +1737,7 @@ fn requires_greater_version_does_not_exist() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of a available matching >1.0.0 and root depends on a>1.0.0, version solving failed.
Because there are no versions of a>1.0.0 and root depends on a>1.0.0, version solving failed.
"###);
});
@ -1790,7 +1794,7 @@ fn requires_less_version_does_not_exist() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of a available matching <2.0.0 and root depends on a<2.0.0, version solving failed.
Because there are no versions of a<2.0.0 and root depends on a<2.0.0, version solving failed.
"###);
});
@ -1974,7 +1978,7 @@ fn requires_transitive_incompatible_with_root_version() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of a available matching <1.0.0 | >1.0.0 and a==1.0.0 depends on b==2.0.0, a depends on b==2.0.0.
Because there are no versions of a<1.0.0 | >1.0.0 and a==1.0.0 depends on b==2.0.0, a depends on b==2.0.0.
And because root depends on a and root depends on b==1.0.0, version solving failed.
"###);
});
@ -2050,8 +2054,8 @@ fn requires_transitive_incompatible_with_transitive() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of b available matching <1.0.0 | >1.0.0 and b==1.0.0 depends on c==2.0.0, b depends on c==2.0.0.
And because a==1.0.0 depends on c==1.0.0 and there is no version of a available matching <1.0.0 | >1.0.0, a *, b * are incompatible.
Because there are no versions of b<1.0.0 | >1.0.0 and b==1.0.0 depends on c==2.0.0, b depends on c==2.0.0.
And because a==1.0.0 depends on c==1.0.0 and there are no versions of a<1.0.0 | >1.0.0, a*, b* are incompatible.
And because root depends on b and root depends on a, version solving failed.
"###);
});
@ -2112,7 +2116,7 @@ fn requires_python_version_does_not_exist() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of Python available matching >=4.0 and a==1.0.0 depends on Python>=4.0, a==1.0.0 is forbidden.
Because there are no versions of Python>=4.0 and a==1.0.0 depends on Python>=4.0, a==1.0.0 is forbidden.
And because root depends on a==1.0.0, version solving failed.
"###);
});
@ -2169,7 +2173,7 @@ fn requires_python_version_less_than_current() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of Python available matching <=3.8 and a==1.0.0 depends on Python<=3.8, a==1.0.0 is forbidden.
Because there are no versions of Python<=3.8 and a==1.0.0 depends on Python<=3.8, a==1.0.0 is forbidden.
And because root depends on a==1.0.0, version solving failed.
"###);
});
@ -2229,7 +2233,7 @@ fn requires_python_version_greater_than_current() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of Python available matching >=3.10 and a==1.0.0 depends on Python>=3.10, a==1.0.0 is forbidden.
Because there are no versions of Python>=3.10 and a==1.0.0 depends on Python>=3.10, a==1.0.0 is forbidden.
And because root depends on a==1.0.0, version solving failed.
"###);
});
@ -2311,7 +2315,7 @@ fn requires_python_version_greater_than_current_many() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of a available matching ==1.0.0 and root depends on a==1.0.0, version solving failed.
Because there is no version of a==1.0.0 and root depends on a==1.0.0, version solving failed.
"###);
});
@ -2447,15 +2451,15 @@ fn requires_python_version_greater_than_current_excluded() -> Result<()> {
----- stderr -----
× No solution found when resolving dependencies:
Because there is no version of Python available matching >=3.10, <3.11 and there is no version of Python available matching >=3.12, Python >=3.10, <3.11 | >=3.12 are incompatible.
And because there is no version of Python available matching >=3.11, <3.12, Python >=3.10 are incompatible.
And because a==2.0.0 depends on Python>=3.10 and there is no version of a available matching >2.0.0, <3.0.0 | >3.0.0, <4.0.0 | >4.0.0, a>=2.0.0, <3.0.0 is forbidden. (1)
Because there are no versions of Python>=3.10, <3.11 and there are no versions of Python>=3.12, Python>=3.10, <3.11 | >=3.12 are incompatible.
And because there are no versions of Python>=3.11, <3.12, Python>=3.10 are incompatible.
And because a==2.0.0 depends on Python>=3.10 and there are no versions of a>2.0.0, <3.0.0 | >3.0.0, <4.0.0 | >4.0.0, a>=2.0.0, <3.0.0 is forbidden. (1)
Because there is no version of Python available matching >=3.11, <3.12 and there is no version of Python available matching >=3.12, Python >=3.11 are incompatible.
Because there are no versions of Python>=3.11, <3.12 and there are no versions of Python>=3.12, Python>=3.11 are incompatible.
And because a==3.0.0 depends on Python>=3.11, a==3.0.0 is forbidden.
And because a>=2.0.0, <3.0.0 is forbidden (1), a>=2.0.0, <4.0.0 is forbidden. (2)
Because there is no version of Python available matching >=3.12 and a==4.0.0 depends on Python>=3.12, a==4.0.0 is forbidden.
Because there are no versions of Python>=3.12 and a==4.0.0 depends on Python>=3.12, a==4.0.0 is forbidden.
And because a>=2.0.0, <4.0.0 is forbidden (2), a>=2.0.0 is forbidden.
And because root depends on a>=2.0.0, version solving failed.
"###);

View File

@ -2314,6 +2314,7 @@ fn sync_editable() -> Result<()> {
"../../scripts/editable-installs/maturin_editable/python/maturin_editable/__init__.py";
let python_version_1 = indoc::indoc! {r"
from .maturin_editable import *
version = 1
"};
fs_err::write(python_source_file, python_version_1)?;
@ -2329,6 +2330,7 @@ fn sync_editable() -> Result<()> {
// Edit the sources.
let python_version_2 = indoc::indoc! {r"
from .maturin_editable import *
version = 2
"};
fs_err::write(python_source_file, python_version_2)?;
@ -2458,7 +2460,7 @@ fn sync_editable_and_registry() -> Result<()> {
Uninstalled 1 package in [TIME]
Installed 1 package in [TIME]
- black==24.1a1
+ black==0.1.0 (from file://[WORKSPACE_DIR]/scripts/editable-installs/black_editable/)
+ black==0.1.0+editable (from file://[WORKSPACE_DIR]/scripts/editable-installs/black_editable/)
"###);
});
@ -2535,7 +2537,7 @@ fn sync_editable_and_registry() -> Result<()> {
Downloaded 1 package in [TIME]
Uninstalled 1 package in [TIME]
Installed 1 package in [TIME]
- black==0.1.0 (from file://[WORKSPACE_DIR]/scripts/editable-installs/black_editable/)
- black==0.1.0+editable (from file://[WORKSPACE_DIR]/scripts/editable-installs/black_editable/)
+ black==23.10.0
warning: The package `black` requires `click >=8.0.0`, but it's not installed.
warning: The package `black` requires `mypy-extensions >=0.4.3`, but it's not installed.

View File

@ -118,9 +118,7 @@ impl SimpleHtml {
{
let requires_python = std::str::from_utf8(requires_python.as_bytes())?;
let requires_python = html_escape::decode_html_entities(requires_python);
let requires_python =
VersionSpecifiers::from_str(&requires_python).map_err(Error::Pep440)?;
Some(requires_python)
Some(VersionSpecifiers::from_str(&requires_python))
} else {
None
};

View File

@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize};
use tempfile::tempfile_in;
use tokio::io::BufWriter;
use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{debug, info_span, instrument, trace, Instrument};
use tracing::{debug, info_span, instrument, trace, warn, Instrument};
use url::Url;
use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename};
@ -465,13 +465,21 @@ impl SimpleMetadata {
DistFilename::WheelFilename(ref inner) => &inner.version,
};
let file = match file.try_into() {
Ok(file) => file,
Err(err) => {
// Ignore files with unparseable version specifiers.
warn!("Skipping file for {package_name}: {err}");
continue;
}
};
match metadata.0.entry(version.clone()) {
std::collections::btree_map::Entry::Occupied(mut entry) => {
entry.get_mut().push(filename, file.into());
entry.get_mut().push(filename, file);
}
std::collections::btree_map::Entry::Vacant(entry) => {
let mut files = VersionFiles::default();
files.push(filename, file.into());
files.push(filename, file);
entry.insert(files);
}
}
@ -514,3 +522,58 @@ impl MediaType {
"application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html;q=0.2, text/html;q=0.01"
}
}
#[cfg(test)]
mod tests {
use std::str::FromStr;
use puffin_normalize::PackageName;
use pypi_types::SimpleJson;
use crate::SimpleMetadata;
#[test]
fn ignore_failing_files() {
// 1.7.7 has an invalid requires-python field (double comma), 1.7.8 is valid
let response = r#"
{
"files": [
{
"core-metadata": false,
"data-dist-info-metadata": false,
"filename": "pyflyby-1.7.7.tar.gz",
"hashes": {
"sha256": "0c4d953f405a7be1300b440dbdbc6917011a07d8401345a97e72cd410d5fb291"
},
"requires-python": ">=2.5, !=3.0.*, !=3.1.*, !=3.2.*, !=3.2.*, !=3.3.*, !=3.4.*,, !=3.5.*, !=3.6.*, <4",
"size": 427200,
"upload-time": "2022-05-19T09:14:36.591835Z",
"url": "https://files.pythonhosted.org/packages/61/93/9fec62902d0b4fc2521333eba047bff4adbba41f1723a6382367f84ee522/pyflyby-1.7.7.tar.gz",
"yanked": false
},
{
"core-metadata": false,
"data-dist-info-metadata": false,
"filename": "pyflyby-1.7.8.tar.gz",
"hashes": {
"sha256": "1ee37474f6da8f98653dbcc208793f50b7ace1d9066f49e2707750a5ba5d53c6"
},
"requires-python": ">=2.5, !=3.0.*, !=3.1.*, !=3.2.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, <4",
"size": 424460,
"upload-time": "2022-08-04T10:42:02.190074Z",
"url": "https://files.pythonhosted.org/packages/ad/39/17180d9806a1c50197bc63b25d0f1266f745fc3b23f11439fccb3d6baa50/pyflyby-1.7.8.tar.gz",
"yanked": false
}
]
}
"#;
let data: SimpleJson = serde_json::from_str(response).unwrap();
let simple_metadata =
SimpleMetadata::from_files(data.files, &PackageName::from_str("pyflyby").unwrap());
let versions: Vec<String> = simple_metadata
.iter()
.map(|(version, _)| version.to_string())
.collect();
assert_eq!(versions, ["1.7.8".to_string()]);
}
}

View File

@ -38,23 +38,24 @@ anstream = { workspace = true }
anyhow = { workspace = true }
chrono = { workspace = true }
clap = { workspace = true, features = ["derive"] }
colored = { workspace = true }
fs-err = { workspace = true }
fs-err = { workspace = true, features = ["tokio"] }
futures = { workspace = true }
indicatif = { workspace = true }
itertools = { workspace = true }
owo-colors = { workspace = true }
petgraph = { workspace = true }
rustc-hash = { workspace = true }
tempfile = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
tracing-durations-export = { version = "0.1.0", features = ["plot"] }
tracing-indicatif = { workspace = true }
tracing-subscriber = { workspace = true }
which = { workspace = true }
url = { workspace = true }
which = { workspace = true }
[target.'cfg(target_os = "windows")'.dependencies]
mimalloc = "0.1.39"
mimalloc = { version = "0.1.39" }
[target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64")))'.dependencies]
tikv-jemallocator = "0.5.4"
tikv-jemallocator = { version = "0.5.4" }

View File

@ -1,19 +1,23 @@
#![allow(clippy::print_stdout, clippy::print_stderr)]
use std::env;
use std::io::IsTerminal;
use std::path::PathBuf;
use std::process::ExitCode;
use std::time::Instant;
use std::time::{Duration, Instant};
use anstream::eprintln;
use anyhow::Result;
use clap::Parser;
use colored::Colorize;
use tracing::debug;
use tracing_durations_export::plot::PlotConfig;
use tracing_durations_export::DurationsLayerBuilder;
use tracing_indicatif::IndicatifLayer;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::EnvFilter;
use owo_colors::OwoColorize;
use resolve_many::ResolveManyArgs;
use crate::build::{build, BuildArgs};
@ -87,6 +91,34 @@ async fn run() -> Result<()> {
#[tokio::main]
async fn main() -> ExitCode {
let (duration_layer, _guard) = if let Ok(location) = env::var("TRACING_DURATIONS_FILE") {
let location = PathBuf::from(location);
if let Some(parent) = location.parent() {
fs_err::tokio::create_dir_all(&parent)
.await
.expect("Failed to create parent of TRACING_DURATIONS_FILE");
}
let plot_config = PlotConfig {
multi_lane: true,
min_length: Some(Duration::from_secs_f32(0.002)),
remove: Some(
["get_cached_with_callback".to_string()]
.into_iter()
.collect(),
),
..PlotConfig::default()
};
let (layer, guard) = DurationsLayerBuilder::default()
.durations_file(&location)
.plot_file(location.with_extension("svg"))
.plot_config(plot_config)
.build()
.expect("Couldn't create TRACING_DURATIONS_FILE files");
(Some(layer), Some(guard))
} else {
(None, None)
};
let indicatif_layer = IndicatifLayer::new();
let indicatif_compatible_writer_layer = tracing_subscriber::fmt::layer()
.with_writer(indicatif_layer.get_stderr_writer())
@ -99,6 +131,7 @@ async fn main() -> ExitCode {
.unwrap()
});
tracing_subscriber::registry()
.with(duration_layer)
.with(filter_layer)
.with(indicatif_compatible_writer_layer)
.with(indicatif_layer)

View File

@ -139,7 +139,7 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
}
// If the file is greater than 5MB, write it to disk; otherwise, keep it in memory.
let byte_size = wheel.file.size.map(|size| ByteSize::b(size as u64));
let byte_size = wheel.file.size.map(ByteSize::b);
let local_wheel = if let Some(byte_size) =
byte_size.filter(|byte_size| *byte_size < ByteSize::mb(5))
{
@ -153,7 +153,14 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
);
// Read into a buffer.
let mut buffer = Vec::with_capacity(wheel.file.size.unwrap_or(0));
let mut buffer = Vec::with_capacity(
wheel
.file
.size
.unwrap_or(0)
.try_into()
.expect("5MB shouldn't be bigger usize::MAX"),
);
let mut reader = tokio::io::BufReader::new(reader.compat());
tokio::io::copy(&mut reader, &mut buffer).await?;

View File

@ -1,6 +1,3 @@
use fs_err as fs;
use tracing::warn;
use distribution_types::CachedWheel;
use platform_tags::Tags;
use puffin_cache::CacheShard;
@ -31,8 +28,8 @@ impl BuiltWheelIndex {
for subdir in directories(&**shard) {
match CachedWheel::from_path(&subdir) {
Ok(None) => {}
Ok(Some(dist_info)) => {
None => {}
Some(dist_info) => {
// Pick the wheel with the highest priority
let compatibility = dist_info.filename.compatibility(tags);
@ -56,18 +53,6 @@ impl BuiltWheelIndex {
candidate = Some(dist_info);
}
}
Err(err) => {
warn!(
"Invalid cache entry at {}, removing. {err}",
subdir.display()
);
if let Err(err) = fs::remove_dir_all(&subdir) {
warn!(
"Failed to remove invalid cache entry at {}: {err}",
subdir.display()
);
}
}
}
}

View File

@ -2,9 +2,7 @@ use std::collections::hash_map::Entry;
use std::collections::BTreeMap;
use std::path::Path;
use fs_err as fs;
use rustc_hash::FxHashMap;
use tracing::warn;
use distribution_types::{CachedRegistryDist, CachedWheel, IndexUrls};
use pep440_rs::Version;
@ -110,8 +108,8 @@ impl<'a> RegistryWheelIndex<'a> {
) {
for wheel_dir in directories(path.as_ref()) {
match CachedWheel::from_path(&wheel_dir) {
Ok(None) => {}
Ok(Some(dist_info)) => {
None => {}
Some(dist_info) => {
let dist_info = dist_info.into_registry_dist();
// Pick the wheel with the highest priority
@ -125,19 +123,6 @@ impl<'a> RegistryWheelIndex<'a> {
versions.insert(dist_info.filename.version.clone(), dist_info);
}
}
Err(err) => {
warn!(
"Invalid cache entry at {}, removing. {err}",
wheel_dir.display()
);
if let Err(err) = fs::remove_dir_all(&wheel_dir) {
warn!(
"Failed to remove invalid cache entry at {}: {err}",
wheel_dir.display()
);
}
}
}
}
}

View File

@ -2,7 +2,7 @@ pub use distribution_database::{DistributionDatabase, DistributionDatabaseError}
pub use download::{BuiltWheel, DiskWheel, InMemoryWheel, LocalWheel};
pub use index::{BuiltWheelIndex, RegistryWheelIndex};
pub use reporter::Reporter;
pub use source_dist::{SourceDistCachedBuilder, SourceDistError};
pub use source::{SourceDistCachedBuilder, SourceDistError};
pub use unzip::Unzip;
mod distribution_database;
@ -11,5 +11,5 @@ mod error;
mod index;
mod locks;
mod reporter;
mod source_dist;
mod source;
mod unzip;

View File

@ -0,0 +1,61 @@
use std::path::PathBuf;
use tracing::warn;
use distribution_filename::WheelFilename;
use platform_tags::Tags;
use puffin_cache::CacheEntry;
use pypi_types::Metadata21;
use crate::source::manifest::{DiskFilenameAndMetadata, Manifest};
/// The information about the wheel we either just built or got from the cache.
#[derive(Debug, Clone)]
pub struct BuiltWheelMetadata {
/// The path to the built wheel.
pub(crate) path: PathBuf,
/// The expected path to the downloaded wheel's entry in the cache.
pub(crate) target: PathBuf,
/// The parsed filename.
pub(crate) filename: WheelFilename,
/// The metadata of the built wheel.
pub(crate) metadata: Metadata21,
}
impl BuiltWheelMetadata {
/// Find a compatible wheel in the cache based on the given manifest.
pub(crate) fn find_in_cache(
tags: &Tags,
manifest: &Manifest,
cache_entry: &CacheEntry,
) -> Option<Self> {
// Find a compatible cache entry in the manifest.
let (filename, cached_dist) = manifest.find_compatible(tags)?;
let metadata = Self::from_cached(filename.clone(), cached_dist.clone(), cache_entry);
// Validate that the wheel exists on disk.
if !metadata.path.is_file() {
warn!(
"Wheel `{}` is present in the manifest, but not on disk",
metadata.path.display()
);
return None;
}
Some(metadata)
}
/// Create a [`BuiltWheelMetadata`] from a cached entry.
pub(crate) fn from_cached(
filename: WheelFilename,
cached_dist: DiskFilenameAndMetadata,
cache_entry: &CacheEntry,
) -> Self {
Self {
path: cache_entry.dir().join(&cached_dist.disk_filename),
target: cache_entry.dir().join(filename.stem()),
filename,
metadata: cached_dist.metadata,
}
}
}

View File

@ -0,0 +1,56 @@
use thiserror::Error;
use tokio::task::JoinError;
use zip::result::ZipError;
use distribution_filename::WheelFilenameError;
use puffin_normalize::PackageName;
/// The caller is responsible for adding the source dist information to the error chain
#[derive(Debug, Error)]
pub enum SourceDistError {
#[error("Building source distributions is disabled")]
NoBuild,
// Network error
#[error("Failed to parse URL: `{0}`")]
UrlParse(String, #[source] url::ParseError),
#[error("Git operation failed")]
Git(#[source] anyhow::Error),
#[error(transparent)]
Request(#[from] reqwest::Error),
#[error(transparent)]
Client(#[from] puffin_client::Error),
// Cache writing error
#[error("Failed to write to source dist cache")]
Io(#[from] std::io::Error),
#[error("Cache deserialization failed")]
Decode(#[from] rmp_serde::decode::Error),
#[error("Cache serialization failed")]
Encode(#[from] rmp_serde::encode::Error),
// Build error
#[error("Failed to build: {0}")]
Build(String, #[source] anyhow::Error),
#[error("Built wheel has an invalid filename")]
WheelFilename(#[from] WheelFilenameError),
#[error("Package metadata name `{metadata}` does not match given name `{given}`")]
NameMismatch {
given: PackageName,
metadata: PackageName,
},
#[error("Failed to parse metadata from built wheel")]
Metadata(#[from] pypi_types::Error),
#[error("Failed to read `dist-info` metadata from built wheel")]
DistInfo(#[from] install_wheel_rs::Error),
#[error("Failed to read zip archive from built wheel")]
Zip(#[from] ZipError),
#[error("Source distribution directory contains neither readable pyproject.toml nor setup.py")]
DirWithoutEntrypoint,
#[error("Failed to extract source distribution: {0}")]
Extract(#[from] puffin_extract::Error),
/// Should not occur; only seen when another task panicked.
#[error("The task executor is broken, did some other task panic?")]
Join(#[from] JoinError),
}

View File

@ -0,0 +1,44 @@
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};
use distribution_filename::WheelFilename;
use platform_tags::Tags;
use pypi_types::Metadata21;
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub(crate) struct Manifest(FxHashMap<WheelFilename, DiskFilenameAndMetadata>);
impl Manifest {
/// Find a compatible wheel in the cache.
pub(crate) fn find_compatible(
&self,
tags: &Tags,
) -> Option<(&WheelFilename, &DiskFilenameAndMetadata)> {
self.0
.iter()
.find(|(filename, _metadata)| filename.is_compatible(tags))
}
}
impl std::ops::Deref for Manifest {
type Target = FxHashMap<WheelFilename, DiskFilenameAndMetadata>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl std::ops::DerefMut for Manifest {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct DiskFilenameAndMetadata {
/// Relative, un-normalized wheel filename in the cache, which can be different than
/// `WheelFilename::to_string`.
pub(crate) disk_filename: String,
/// The [`Metadata21`] of the wheel.
pub(crate) metadata: Metadata21,
}

View File

@ -8,18 +8,13 @@ use anyhow::Result;
use fs_err::tokio as fs;
use futures::TryStreamExt;
use reqwest::Response;
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};
use tempfile::TempDir;
use thiserror::Error;
use tokio::task::JoinError;
use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{debug, info_span, instrument, warn, Instrument};
use url::Url;
use zip::result::ZipError;
use zip::ZipArchive;
use distribution_filename::{WheelFilename, WheelFilenameError};
use distribution_filename::WheelFilename;
use distribution_types::{
DirectArchiveUrl, DirectGitUrl, Dist, GitSourceDist, LocalEditable, Name, PathSourceDist,
RemoteSource, SourceDist,
@ -30,147 +25,18 @@ use puffin_cache::{CacheBucket, CacheEntry, CacheShard, CachedByTimestamp, Wheel
use puffin_client::{CachedClient, CachedClientError, DataWithCachePolicy};
use puffin_fs::{write_atomic, LockedFile};
use puffin_git::{Fetch, GitSource};
use puffin_normalize::PackageName;
use puffin_traits::{BuildContext, BuildKind, SourceBuildTrait};
use pypi_types::Metadata21;
use crate::reporter::Facade;
use crate::source::built_wheel_metadata::BuiltWheelMetadata;
pub use crate::source::error::SourceDistError;
use crate::source::manifest::{DiskFilenameAndMetadata, Manifest};
use crate::Reporter;
/// The caller is responsible for adding the source dist information to the error chain
#[derive(Debug, Error)]
pub enum SourceDistError {
#[error("Building source distributions is disabled")]
NoBuild,
// Network error
#[error("Failed to parse URL: `{0}`")]
UrlParse(String, #[source] url::ParseError),
#[error("Git operation failed")]
Git(#[source] anyhow::Error),
#[error(transparent)]
Request(#[from] reqwest::Error),
#[error(transparent)]
Client(#[from] puffin_client::Error),
// Cache writing error
#[error("Failed to write to source dist cache")]
Io(#[from] std::io::Error),
#[error("Cache deserialization failed")]
Decode(#[from] rmp_serde::decode::Error),
#[error("Cache serialization failed")]
Encode(#[from] rmp_serde::encode::Error),
// Build error
#[error("Failed to build: {0}")]
Build(String, #[source] anyhow::Error),
#[error("Built wheel has an invalid filename")]
WheelFilename(#[from] WheelFilenameError),
#[error("Package metadata name `{metadata}` does not match given name `{given}`")]
NameMismatch {
given: PackageName,
metadata: PackageName,
},
#[error("Failed to parse metadata from built wheel")]
Metadata(#[from] pypi_types::Error),
#[error("Failed to read `dist-info` metadata from built wheel")]
DistInfo(#[from] install_wheel_rs::Error),
#[error("Failed to read zip archive from built wheel")]
Zip(#[from] ZipError),
#[error("Source distribution directory contains neither readable pyproject.toml nor setup.py")]
DirWithoutEntrypoint,
#[error("Failed to extract source distribution: {0}")]
Extract(#[from] puffin_extract::Error),
/// Should not occur; only seen when another task panicked.
#[error("The task executor is broken, did some other task panic?")]
Join(#[from] JoinError),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct DiskFilenameAndMetadata {
/// Relative, un-normalized wheel filename in the cache, which can be different than
/// `WheelFilename::to_string`.
disk_filename: String,
metadata: Metadata21,
}
/// The information about the wheel we either just built or got from the cache.
#[derive(Debug, Clone)]
pub struct BuiltWheelMetadata {
/// The path to the built wheel.
pub path: PathBuf,
/// The expected path to the downloaded wheel's entry in the cache.
pub target: PathBuf,
/// The parsed filename.
pub filename: WheelFilename,
/// The metadata of the built wheel.
pub metadata: Metadata21,
}
impl BuiltWheelMetadata {
/// Find a compatible wheel in the cache based on the given manifest.
fn find_in_cache(tags: &Tags, manifest: &Manifest, cache_entry: &CacheEntry) -> Option<Self> {
// Find a compatible cache entry in the manifest.
let (filename, cached_dist) = manifest.find_compatible(tags)?;
let metadata = Self::from_cached(filename.clone(), cached_dist.clone(), cache_entry);
// Validate that the wheel exists on disk.
if !metadata.path.is_file() {
warn!(
"Wheel `{}` is present in the manifest, but not on disk",
metadata.path.display()
);
return None;
}
Some(metadata)
}
/// Create a [`BuiltWheelMetadata`] from a cached entry.
fn from_cached(
filename: WheelFilename,
cached_dist: DiskFilenameAndMetadata,
cache_entry: &CacheEntry,
) -> Self {
Self {
path: cache_entry.dir().join(&cached_dist.disk_filename),
target: cache_entry.dir().join(filename.stem()),
filename,
metadata: cached_dist.metadata,
}
}
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
struct Manifest(FxHashMap<WheelFilename, DiskFilenameAndMetadata>);
impl Manifest {
/// Initialize a [`Manifest`] from an iterator over entries.
fn from_iter(iter: impl IntoIterator<Item = (WheelFilename, DiskFilenameAndMetadata)>) -> Self {
Self(iter.into_iter().collect())
}
/// Find a compatible wheel in the cache.
fn find_compatible(&self, tags: &Tags) -> Option<(&WheelFilename, &DiskFilenameAndMetadata)> {
self.0
.iter()
.find(|(filename, _metadata)| filename.is_compatible(tags))
}
}
impl std::ops::Deref for Manifest {
type Target = FxHashMap<WheelFilename, DiskFilenameAndMetadata>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl std::ops::DerefMut for Manifest {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
mod built_wheel_metadata;
mod error;
mod manifest;
/// Fetch and build a source distribution from a remote source, or from a local cache.
pub struct SourceDistCachedBuilder<'a, T: BuildContext> {
@ -275,53 +141,31 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
) -> Result<BuiltWheelMetadata, SourceDistError> {
let cache_entry = cache_shard.entry(METADATA);
let download_and_build = |response| {
let download = |response| {
async {
// At this point, we're seeing a new or updated source distribution; delete all
// wheels, and rebuild.
// wheels, and redownload.
match fs::remove_dir_all(&cache_entry.dir()).await {
Ok(()) => debug!("Cleared built wheels and metadata for {source_dist}"),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => (),
Err(err) => return Err(err.into()),
}
debug!("Downloading and building source distribution: {source_dist}");
let task = self
.reporter
.as_ref()
.map(|reporter| reporter.on_build_start(source_dist));
debug!("Downloading source distribution: {source_dist}");
// Download the source distribution.
let source_dist_entry = cache_shard.entry(filename);
let cache_dir = self
.persist_source_dist_url(response, source_dist, filename, &source_dist_entry)
self.persist_source_dist_url(response, source_dist, filename, &source_dist_entry)
.await?;
// Build the source distribution.
let (disk_filename, wheel_filename, metadata) = self
.build_source_dist(source_dist, cache_dir, subdirectory, &cache_entry)
.await?;
if let Some(task) = task {
if let Some(reporter) = self.reporter.as_ref() {
reporter.on_build_complete(source_dist, task);
}
}
Ok(Manifest::from_iter([(
wheel_filename,
DiskFilenameAndMetadata {
disk_filename,
metadata,
},
)]))
Ok(Manifest::default())
}
.instrument(info_span!("download_and_build", source_dist = %source_dist))
.instrument(info_span!("download", source_dist = %source_dist))
};
let req = self.cached_client.uncached().get(url.clone()).build()?;
let manifest = self
.cached_client
.get_cached_with_callback(req, &cache_entry, download_and_build)
.get_cached_with_callback(req, &cache_entry, download)
.await
.map_err(|err| match err {
CachedClientError::Callback(err) => err,
@ -329,10 +173,10 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
})?;
// If the cache contains a compatible wheel, return it.
if let Some(metadata) =
if let Some(built_wheel) =
BuiltWheelMetadata::find_in_cache(self.tags, &manifest, &cache_entry)
{
return Ok(metadata);
return Ok(built_wheel);
}
// At this point, we're seeing cached metadata (as in, we have an up-to-date source
@ -342,23 +186,15 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
.as_ref()
.map(|reporter| reporter.on_build_start(source_dist));
// Start by downloading the source distribution.
let response = self
.cached_client
.uncached()
.get(url.clone())
.send()
.await
.map_err(puffin_client::Error::RequestMiddlewareError)?;
let source_dist_entry = cache_shard.entry(filename);
let cache_dir = self
.persist_source_dist_url(response, source_dist, filename, &source_dist_entry)
.await?;
// Build the source distribution.
let source_dist_entry = cache_shard.entry(filename);
let (disk_filename, wheel_filename, metadata) = self
.build_source_dist(source_dist, cache_dir, subdirectory, &cache_entry)
.build_source_dist(
source_dist,
source_dist_entry.path(),
subdirectory,
&cache_entry,
)
.await?;
if let Some(task) = task {
@ -440,10 +276,10 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
.unwrap_or_default();
// If the cache contains a compatible wheel, return it.
if let Some(metadata) =
if let Some(built_wheel) =
BuiltWheelMetadata::find_in_cache(self.tags, &manifest, &cache_entry)
{
return Ok(metadata);
return Ok(built_wheel);
}
// Otherwise, we need to build a wheel.
@ -516,10 +352,10 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
let mut manifest = Self::read_metadata(&cache_entry).await?.unwrap_or_default();
// If the cache contains a compatible wheel, return it.
if let Some(metadata) =
if let Some(built_wheel) =
BuiltWheelMetadata::find_in_cache(self.tags, &manifest, &cache_entry)
{
return Ok(metadata);
return Ok(built_wheel);
}
let task = self
@ -799,32 +635,3 @@ fn read_metadata(
let dist_info = read_dist_info(filename, &mut archive)?;
Ok(Metadata21::parse(&dist_info)?)
}
trait SourceDistReporter: Send + Sync {
/// Callback to invoke when a repository checkout begins.
fn on_checkout_start(&self, url: &Url, rev: &str) -> usize;
/// Callback to invoke when a repository checkout completes.
fn on_checkout_complete(&self, url: &Url, rev: &str, index: usize);
}
/// A facade for converting from [`Reporter`] to [`puffin_git::Reporter`].
struct Facade {
reporter: Arc<dyn Reporter>,
}
impl From<Arc<dyn Reporter>> for Facade {
fn from(reporter: Arc<dyn Reporter>) -> Self {
Self { reporter }
}
}
impl puffin_git::Reporter for Facade {
fn on_checkout_start(&self, url: &Url, rev: &str) -> usize {
self.reporter.on_checkout_start(url, rev)
}
fn on_checkout_complete(&self, url: &Url, rev: &str, index: usize) {
self.reporter.on_checkout_complete(url, rev, index);
}
}

View File

@ -87,9 +87,8 @@ impl<'a, Context: BuildContext + Send + Sync> Downloader<'a, Context> {
in_flight: &OnceMap<PathBuf, Result<CachedDist, String>>,
) -> Result<Vec<CachedDist>, Error> {
// Sort the distributions by size.
distributions.sort_unstable_by_key(|distribution| {
Reverse(distribution.size().unwrap_or(usize::MAX))
});
distributions
.sort_unstable_by_key(|distribution| Reverse(distribution.size().unwrap_or(u64::MAX)));
let wheels = self
.download_stream(distributions, in_flight)

View File

@ -1,5 +1,6 @@
use anyhow::{Context, Error, Result};
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
use tracing::instrument;
use distribution_types::CachedDist;
use puffin_interpreter::Virtualenv;
@ -36,6 +37,7 @@ impl<'a> Installer<'a> {
}
/// Install a set of wheels into a Python virtual environment.
#[instrument(skip_all, fields(num_wheels = %wheels.len()))]
pub fn install(self, wheels: &[CachedDist]) -> Result<()> {
tokio::task::block_in_place(|| {
wheels.par_iter().try_for_each(|wheel| {

View File

@ -38,10 +38,10 @@ impl<'a> SitePackages<'a> {
for entry in fs::read_dir(venv.site_packages())? {
let entry = entry?;
if entry.file_type()?.is_dir() {
let Some(dist_info) =
InstalledDist::try_from_path(&entry.path()).with_context(|| {
format!("Failed to read metadata: from {}", entry.path().display())
})?
let path = entry.path();
let Some(dist_info) = InstalledDist::try_from_path(&path)
.with_context(|| format!("Failed to read metadata: from {}", path.display()))?
else {
continue;
};
@ -55,7 +55,7 @@ impl<'a> SitePackages<'a> {
"Found duplicate package in environment: {} ({} vs. {})",
existing.name(),
existing.path().display(),
entry.path().display()
path.display()
);
}
@ -67,7 +67,7 @@ impl<'a> SitePackages<'a> {
"Found duplicate editable in environment: {} ({} vs. {})",
existing.name(),
existing.path().display(),
entry.path().display()
path.display()
);
}
}

View File

@ -32,17 +32,18 @@ puffin-traits = { path = "../puffin-traits" }
pypi-types = { path = "../pypi-types" }
requirements-txt = { path = "../requirements-txt" }
anstream = { workspace = true }
anyhow = { workspace = true }
bitflags = { workspace = true }
chrono = { workspace = true }
clap = { workspace = true, features = ["derive"], optional = true }
colored = { workspace = true }
derivative = { workspace = true }
fs-err = { workspace = true, features = ["tokio"] }
futures = { workspace = true }
http-cache-semantics = { workspace = true }
itertools = { workspace = true }
once_cell = { workspace = true }
owo-colors = { workspace = true }
petgraph = { workspace = true }
pubgrub = { workspace = true }
reqwest = { workspace = true }

View File

@ -256,7 +256,7 @@ impl<'a> Candidate<'a> {
self.file.install()
}
/// If the candidate doesn't the given requirement, return the version specifiers.
/// If the candidate doesn't match the given requirement, return the version specifiers.
pub(crate) fn validate(&self, requirement: &PythonRequirement) -> Option<&VersionSpecifiers> {
// Validate against the _installed_ file. It's fine if the _resolved_ file is incompatible,
// since it could be an incompatible wheel. (If the resolved file is an incompatible source

View File

@ -1,13 +1,15 @@
use crate::candidate_selector::CandidateSelector;
use crate::prerelease_mode::PreReleaseStrategy;
use colored::Colorize;
use std::borrow::Cow;
use derivative::Derivative;
use owo_colors::OwoColorize;
use pubgrub::range::Range;
use pubgrub::report::{DerivationTree, External, ReportFormatter};
use pubgrub::term::Term;
use pubgrub::type_aliases::Map;
use rustc_hash::{FxHashMap, FxHashSet};
use std::borrow::Cow;
use crate::candidate_selector::CandidateSelector;
use crate::prerelease_mode::PreReleaseStrategy;
use super::{PubGrubPackage, PubGrubVersion};
@ -31,9 +33,11 @@ impl ReportFormatter<PubGrubPackage, Range<PubGrubVersion>> for PubGrubReportFor
External::NoVersions(package, set) => {
let set = self.simplify_set(set, package);
if set.as_ref() == &Range::full() {
format!("there is no available version for {package}")
format!("there are no versions of {package}")
} else if set.as_singleton().is_some() {
format!("there is no version of {package}{set}")
} else {
format!("there is no version of {package} available matching {set}")
format!("there are no versions of {package}{set}")
}
}
External::UnavailableDependencies(package, set) => {
@ -41,7 +45,7 @@ impl ReportFormatter<PubGrubPackage, Range<PubGrubVersion>> for PubGrubReportFor
if set.as_ref() == &Range::full() {
format!("dependencies of {package} are unavailable")
} else {
format!("dependencies of {package} at version {set} are unavailable")
format!("dependencies of {package}{set} are unavailable")
}
}
External::UnusableDependencies(package, set, reason) => {
@ -123,7 +127,10 @@ impl ReportFormatter<PubGrubPackage, Range<PubGrubVersion>> for PubGrubReportFor
&External::FromDependencyOf((*p2).clone(), r2.clone(), (*p1).clone(), r1.clone()),
),
slice => {
let str_terms: Vec<_> = slice.iter().map(|(p, t)| format!("{p} {t}")).collect();
let str_terms: Vec<_> = slice
.iter()
.map(|(p, t)| format!("{p}{}", PubGrubTerm::from_term((*t).clone())))
.collect();
str_terms.join(", ") + " are incompatible"
}
}
@ -153,39 +160,57 @@ impl PubGrubReportFormatter<'_> {
derivation_tree: &DerivationTree<PubGrubPackage, Range<PubGrubVersion>>,
selector: &CandidateSelector,
) -> FxHashSet<PubGrubHint> {
/// Returns `true` if pre-releases were allowed for a package.
fn allowed_prerelease(package: &PubGrubPackage, selector: &CandidateSelector) -> bool {
match selector.prerelease_strategy() {
PreReleaseStrategy::Disallow => false,
PreReleaseStrategy::Allow => true,
PreReleaseStrategy::IfNecessary => false,
PreReleaseStrategy::Explicit(packages) => {
if let PubGrubPackage::Package(package, ..) = package {
packages.contains(package)
} else {
false
}
}
PreReleaseStrategy::IfNecessaryOrExplicit(packages) => {
if let PubGrubPackage::Package(package, ..) = package {
packages.contains(package)
} else {
false
}
}
}
}
let mut hints = FxHashSet::default();
match derivation_tree {
DerivationTree::External(external) => match external {
External::NoVersions(package, set) => {
// Determine whether a pre-release marker appeared in the version requirements.
if set.bounds().any(PubGrubVersion::any_prerelease) {
// Determine whether pre-releases were allowed for this package.
let allowed_prerelease = match selector.prerelease_strategy() {
PreReleaseStrategy::Disallow => false,
PreReleaseStrategy::Allow => true,
PreReleaseStrategy::IfNecessary => false,
PreReleaseStrategy::Explicit(packages) => {
if let PubGrubPackage::Package(package, ..) = package {
packages.contains(package)
} else {
false
}
}
PreReleaseStrategy::IfNecessaryOrExplicit(packages) => {
if let PubGrubPackage::Package(package, ..) = package {
packages.contains(package)
} else {
false
}
}
};
if !allowed_prerelease {
hints.insert(PubGrubHint::NoVersionsWithPreRelease {
// A pre-release marker appeared in the version requirements.
if !allowed_prerelease(package, selector) {
hints.insert(PubGrubHint::PreReleaseRequested {
package: package.clone(),
range: self.simplify_set(set, package).into_owned(),
});
}
} else if let Some(version) =
self.available_versions.get(package).and_then(|versions| {
versions
.iter()
.rev()
.filter(|version| version.any_prerelease())
.find(|version| set.contains(version))
})
{
// There are pre-release versions available for the package.
if !allowed_prerelease(package, selector) {
hints.insert(PubGrubHint::PreReleaseAvailable {
package: package.clone(),
version: version.clone(),
});
}
}
}
External::NotRoot(..) => {}
@ -205,9 +230,17 @@ impl PubGrubReportFormatter<'_> {
#[derive(Derivative, Debug, Clone)]
#[derivative(Hash, PartialEq, Eq)]
pub(crate) enum PubGrubHint {
/// A package was requested with a pre-release marker, but pre-releases weren't enabled for
/// that package.
NoVersionsWithPreRelease {
/// There are pre-release versions available for a package, but pre-releases weren't enabled
/// for that package.
///
PreReleaseAvailable {
package: PubGrubPackage,
#[derivative(PartialEq = "ignore", Hash = "ignore")]
version: PubGrubVersion,
},
/// A requirement included a pre-release marker, but pre-releases weren't enabled for that
/// package.
PreReleaseRequested {
package: PubGrubPackage,
#[derivative(PartialEq = "ignore", Hash = "ignore")]
range: Range<PubGrubVersion>,
@ -217,16 +250,52 @@ pub(crate) enum PubGrubHint {
impl std::fmt::Display for PubGrubHint {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
PubGrubHint::NoVersionsWithPreRelease { package, range } => {
PubGrubHint::PreReleaseAvailable { package, version } => {
write!(
f,
"{}{} Pre-releases are available for {} in the requested range (e.g., {}), but pre-releases weren't enabled (try: `--prerelease=allow`)",
"hint".bold().cyan(),
":".bold(),
package.bold(),
version.bold()
)
}
PubGrubHint::PreReleaseRequested { package, range } => {
write!(
f,
"{}{} {} was requested with a pre-release marker (e.g., {}), but pre-releases weren't enabled (try: `--prerelease=allow`)",
"hint".bold().cyan(),
":".bold(),
format!("{package}").bold(),
format!("{range}").bold(),
package.bold(),
range.bold()
)
}
}
}
}
/// A derivative of the [Term] type with custom formatting.
struct PubGrubTerm {
inner: Term<Range<PubGrubVersion>>,
}
impl std::fmt::Display for PubGrubTerm {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match &self.inner {
Term::Positive(set) => write!(f, "{set}"),
Term::Negative(set) => {
if let Some(version) = set.as_singleton() {
write!(f, "!={version}")
} else {
write!(f, "!( {set} )")
}
}
}
}
}
impl PubGrubTerm {
fn from_term(term: Term<Range<PubGrubVersion>>) -> PubGrubTerm {
PubGrubTerm { inner: term }
}
}

View File

@ -1,7 +1,7 @@
use std::hash::BuildHasherDefault;
use anyhow::Result;
use colored::Colorize;
use owo_colors::OwoColorize;
use petgraph::visit::EdgeRef;
use petgraph::Direction;
use pubgrub::range::Range;

View File

@ -114,6 +114,13 @@ async fn resolve(
Ok(resolver.resolve().await?)
}
macro_rules! assert_snapshot {
($value:expr, @$snapshot:literal) => {
let snapshot = anstream::adapter::strip_str(&format!("{}", $value)).to_string();
insta::assert_snapshot!(&snapshot, @$snapshot)
};
}
#[tokio::test]
async fn black() -> Result<()> {
let manifest = Manifest::simple(vec![Requirement::from_str("black<=23.9.1").unwrap()]);
@ -125,7 +132,7 @@ async fn black() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
black==23.9.1
click==8.1.7
# via black
@ -155,7 +162,7 @@ async fn black_colorama() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
black==23.9.1
click==8.1.7
# via black
@ -187,7 +194,7 @@ async fn black_tensorboard() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
black==23.9.1
click==8.1.7
# via black
@ -215,7 +222,7 @@ async fn black_python_310() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_310, &TAGS_310).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
black==23.9.1
click==8.1.7
# via black
@ -256,7 +263,7 @@ async fn black_mypy_extensions() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
black==23.9.1
click==8.1.7
# via black
@ -293,7 +300,7 @@ async fn black_mypy_extensions_extra() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
black==23.9.1
click==8.1.7
# via black
@ -330,7 +337,7 @@ async fn black_flake8() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
black==23.9.1
click==8.1.7
# via black
@ -358,7 +365,7 @@ async fn black_lowest() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
black==22.1.0
click==8.0.0
# via black
@ -386,7 +393,7 @@ async fn black_lowest_direct() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
black==22.1.0
click==8.1.7
# via black
@ -421,7 +428,7 @@ async fn black_respect_preference() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
black==23.9.0
click==8.1.7
# via black
@ -456,7 +463,7 @@ async fn black_ignore_preference() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
black==23.9.1
click==8.1.7
# via black
@ -486,7 +493,11 @@ async fn black_disallow_prerelease() -> Result<()> {
.await
.unwrap_err();
insta::assert_display_snapshot!(err, @"Because there is no version of black available matching <=20.0 and root depends on black<=20.0, version solving failed.");
assert_snapshot!(err, @r###"
Because there are no versions of black<=20.0 and root depends on black<=20.0, version solving failed.
hint: Pre-releases are available for black in the requested range (e.g., 19.10b0), but pre-releases weren't enabled (try: `--prerelease=allow`)
"###);
Ok(())
}
@ -504,7 +515,11 @@ async fn black_allow_prerelease_if_necessary() -> Result<()> {
.await
.unwrap_err();
insta::assert_display_snapshot!(err, @"Because there is no version of black available matching <=20.0 and root depends on black<=20.0, version solving failed.");
assert_snapshot!(err, @r###"
Because there are no versions of black<=20.0 and root depends on black<=20.0, version solving failed.
hint: Pre-releases are available for black in the requested range (e.g., 19.10b0), but pre-releases weren't enabled (try: `--prerelease=allow`)
"###);
Ok(())
}
@ -520,7 +535,7 @@ async fn pylint_disallow_prerelease() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
astroid==3.0.1
# via pylint
isort==5.12.0
@ -544,7 +559,7 @@ async fn pylint_allow_prerelease() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
astroid==3.0.1
# via pylint
isort==6.0.0b2
@ -571,7 +586,7 @@ async fn pylint_allow_explicit_prerelease_without_marker() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
astroid==3.0.1
# via pylint
isort==5.12.0
@ -598,7 +613,7 @@ async fn pylint_allow_explicit_prerelease_with_marker() -> Result<()> {
let resolution = resolve(manifest, options, &MARKERS_311, &TAGS_311).await?;
insta::assert_display_snapshot!(resolution, @r###"
assert_snapshot!(resolution, @r###"
astroid==3.0.1
# via pylint
isort==6.0.0b2
@ -626,8 +641,8 @@ async fn msgraph_sdk() -> Result<()> {
.await
.unwrap_err();
insta::assert_display_snapshot!(err, @r###"
Because there is no version of msgraph-core available matching >=1.0.0a2 and msgraph-sdk==1.0.0 depends on msgraph-core>=1.0.0a2, msgraph-sdk==1.0.0 is forbidden.
assert_snapshot!(err, @r###"
Because there are no versions of msgraph-core>=1.0.0a2 and msgraph-sdk==1.0.0 depends on msgraph-core>=1.0.0a2, msgraph-sdk==1.0.0 is forbidden.
And because root depends on msgraph-sdk==1.0.0, version solving failed.
hint: msgraph-core was requested with a pre-release marker (e.g., >=1.0.0a2), but pre-releases weren't enabled (try: `--prerelease=allow`)

View File

@ -14,6 +14,6 @@ workspace = true
[dependencies]
anstream = { workspace = true }
colored = { workspace = true }
once_cell = { workspace = true }
owo-colors = { workspace = true }
rustc-hash = { workspace = true }

View File

@ -4,11 +4,9 @@ use std::sync::Mutex;
// macro hygiene: The user might not have direct dependencies on those crates
#[doc(hidden)]
pub use anstream;
#[doc(hidden)]
pub use colored;
use once_cell::sync::Lazy;
#[doc(hidden)]
pub use owo_colors;
use rustc_hash::FxHashSet;
/// Whether user-facing warnings are enabled.
@ -24,7 +22,7 @@ pub fn enable() {
macro_rules! warn_user {
($($arg:tt)*) => {
use $crate::anstream::eprintln;
use $crate::colored::Colorize;
use $crate::owo_colors::OwoColorize;
if $crate::ENABLED.load(std::sync::atomic::Ordering::SeqCst) {
let message = format!("{}", format_args!($($arg)*));
@ -42,14 +40,13 @@ pub static WARNINGS: Lazy<Mutex<FxHashSet<String>>> = Lazy::new(Mutex::default);
macro_rules! warn_user_once {
($($arg:tt)*) => {
use $crate::anstream::eprintln;
use $crate::colored::Colorize;
use $crate::owo_colors::OwoColorize;
if $crate::ENABLED.load(std::sync::atomic::Ordering::SeqCst) {
if let Ok(mut states) = $crate::WARNINGS.lock() {
let message = format!("{}", format_args!($($arg)*));
let formatted = message.bold();
if states.insert(message) {
eprintln!("{}{} {formatted}", "warning".yellow().bold(), ":".bold());
if states.insert(message.clone()) {
eprintln!("{}{} {}", "warning".yellow().bold(), ":".bold(), message.bold());
}
}
}

View File

@ -3,10 +3,10 @@ use std::str::FromStr;
use once_cell::sync::Lazy;
use regex::Regex;
use serde::{de, Deserialize, Deserializer, Serialize};
use tracing::warn;
use pep440_rs::{VersionSpecifiers, VersionSpecifiersParseError};
use pep508_rs::{Pep508Error, Requirement};
use puffin_warnings::warn_user_once;
/// Ex) `>=7.2.0<8.0.0`
static MISSING_COMMA: Lazy<Regex> = Lazy::new(|| Regex::new(r"(\d)([<>=~^!])").unwrap());
@ -62,7 +62,7 @@ fn parse_with_fixups<Err, T: FromStr<Err = Err>>(input: &str, type_name: &str) -
}
if let Ok(requirement) = T::from_str(&patched_input) {
warn_user_once!(
warn!(
"Fixing invalid {type_name} by {} (before: `{input}`; after: `{patched_input}`)",
messages.join(", ")
);

View File

@ -1,9 +1,9 @@
use std::str::FromStr;
use chrono::{DateTime, Utc};
use serde::{de, Deserialize, Deserializer, Serialize};
use serde::{Deserialize, Deserializer, Serialize};
use pep440_rs::VersionSpecifiers;
use pep440_rs::{VersionSpecifiers, VersionSpecifiersParseError};
use crate::lenient_requirement::LenientVersionSpecifiers;
@ -23,11 +23,12 @@ pub struct File {
pub dist_info_metadata: Option<DistInfoMetadata>,
pub filename: String,
pub hashes: Hashes,
/// Note: Deserialized with [`LenientVersionSpecifiers`] since there are a number of invalid
/// versions on pypi
/// There are a number of invalid specifiers on pypi, so we first try to parse it into a [`VersionSpecifiers`]
/// according to spec (PEP 440), then a [`LenientVersionSpecifiers`] with fixup for some common problems and if this
/// still fails, we skip the file when creating a version map.
#[serde(default, deserialize_with = "deserialize_version_specifiers_lenient")]
pub requires_python: Option<VersionSpecifiers>,
pub size: Option<usize>,
pub requires_python: Option<Result<VersionSpecifiers, VersionSpecifiersParseError>>,
pub size: Option<u64>,
pub upload_time: Option<DateTime<Utc>>,
pub url: String,
pub yanked: Option<Yanked>,
@ -35,7 +36,7 @@ pub struct File {
fn deserialize_version_specifiers_lenient<'de, D>(
deserializer: D,
) -> Result<Option<VersionSpecifiers>, D::Error>
) -> Result<Option<Result<VersionSpecifiers, VersionSpecifiersParseError>>, D::Error>
where
D: Deserializer<'de>,
{
@ -43,8 +44,9 @@ where
let Some(string) = maybe_string else {
return Ok(None);
};
let lenient = LenientVersionSpecifiers::from_str(&string).map_err(de::Error::custom)?;
Ok(Some(lenient.into()))
Ok(Some(
LenientVersionSpecifiers::from_str(&string).map(Into::into),
))
}
#[derive(Debug, Clone, Serialize, Deserialize)]

View File

@ -10,7 +10,8 @@ To set up the required environment, run:
cargo build --release
./target/release/puffin venv
./target/release/puffin pip-sync ./scripts/requirements.txt
./target/release/puffin pip-sync ./scripts/bench/requirements.txt
source .venv/bin/activate
Example usage:

33
scripts/benchmarks/venv.sh Executable file
View File

@ -0,0 +1,33 @@
#!/usr/bin/env bash
###
# Benchmark the virtualenv initialization against `virtualenv`.
#
# Example usage:
#
# ./scripts/benchmarks/venv.sh ./scripts/benchmarks/requirements.txt
###
set -euxo pipefail
###
# Create a virtual environment without seed packages.
###
hyperfine --runs 20 --warmup 3 \
--prepare "rm -rf .venv" \
"./target/release/puffin venv --no-cache" \
--prepare "rm -rf .venv" \
"virtualenv --without-pip .venv" \
--prepare "rm -rf .venv" \
"python -m venv --without-pip .venv"
###
# Create a virtual environment with seed packages.
#
# TODO(charlie): Support seed packages in `puffin venv`.
###
hyperfine --runs 20 --warmup 3 \
--prepare "rm -rf .venv" \
"virtualenv .venv" \
--prepare "rm -rf .venv" \
"python -m venv .venv"

View File

@ -61,7 +61,7 @@ def resolve_puffin(targets: list[str], venv: Path, profile: str = "dev") -> list
output = check_output(
[
project_root.joinpath("target").joinpath(target_profile).joinpath("puffin-dev"),
"resolve-cli",
"resolve",
"--format",
"expanded",
*targets,

View File

@ -0,0 +1,162 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm-project.org/#use-with-ide
.pdm.toml
.pdm-python
.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/

View File

@ -1,12 +1,18 @@
[tool.poetry]
[project]
name = "black"
version = "0.1.0"
description = ""
authors = ["konstin <konstin@mailbox.org>"]
[tool.poetry.dependencies]
python = "^3.10"
description = "Default template for PDM package"
authors = [
{name = "konstin", email = "konstin@mailbox.org"},
]
dependencies = []
requires-python = ">=3.11,<3.13"
license = {text = "MIT"}
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
requires = ["pdm-backend"]
build-backend = "pdm.backend"
[tool.pdm]
package-type = "library"