Merge branch 'main' into zb/nextest

This commit is contained in:
Zanie 2024-01-11 12:29:50 -06:00
commit 5d6430597d
20 changed files with 190 additions and 265 deletions

151
Cargo.lock generated
View File

@ -282,9 +282,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "base64"
version = "0.21.5"
version = "0.21.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9"
checksum = "c79fed4cdb43e993fcdadc7e58a09fd0e3e649c4436fa11da71c9f1f3ee7feb9"
[[package]]
name = "bench"
@ -519,9 +519,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.4.13"
version = "4.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52bdc885e4cacc7f7c9eedc1ef6da641603180c783c41a15c264944deeaab642"
checksum = "33e92c5c1a78c62968ec57dbc2440366a2d6e5a23faf829970ff1585dc6b18e2"
dependencies = [
"clap_builder",
"clap_derive",
@ -529,9 +529,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.4.12"
version = "4.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9"
checksum = "f4323769dc8a61e2c39ad7dc26f6f2800524691a44d74fe3d1071a5c24db6370"
dependencies = [
"anstream",
"anstyle",
@ -580,15 +580,15 @@ checksum = "4ec6d3da8e550377a85339063af6e3735f4b1d9392108da4e083a1b3b9820288"
[[package]]
name = "console"
version = "0.15.7"
version = "0.15.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8"
checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
dependencies = [
"encode_unicode",
"lazy_static",
"libc",
"unicode-width",
"windows-sys 0.45.0",
"windows-sys 0.52.0",
]
[[package]]
@ -681,34 +681,28 @@ dependencies = [
[[package]]
name = "crossbeam-deque"
version = "0.8.4"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751"
checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.17"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"autocfg",
"cfg-if 1.0.0",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.18"
version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c"
dependencies = [
"cfg-if 1.0.0",
]
checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
[[package]]
name = "crunchy"
@ -1029,9 +1023,9 @@ checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
[[package]]
name = "futures-lite"
version = "2.1.0"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aeee267a1883f7ebef3700f262d2d54de95dfaf38189015a74fdc4e0c7ad8143"
checksum = "445ba825b27408685aaecefd65178908c36c6e96aaf6d8599419d46e624192ba"
dependencies = [
"fastrand",
"futures-core",
@ -1093,9 +1087,9 @@ dependencies = [
[[package]]
name = "getrandom"
version = "0.2.11"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"
dependencies = [
"cfg-if 1.0.0",
"js-sys",
@ -1189,9 +1183,9 @@ dependencies = [
[[package]]
name = "h2"
version = "0.3.22"
version = "0.3.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178"
checksum = "b553656127a00601c8ae5590fcfdc118e4083a7924b6cf4ffc1ea4b99dc429d7"
dependencies = [
"bytes",
"fnv",
@ -1399,9 +1393,9 @@ dependencies = [
[[package]]
name = "ignore"
version = "0.4.21"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "747ad1b4ae841a78e8aba0d63adbfbeaea26b517b63705d47856b73015d27060"
checksum = "b46810df39e66e925525d6e38ce1e7f6e1d208f72dc39757880fcb66e2c58af1"
dependencies = [
"crossbeam-deque",
"globset",
@ -1613,9 +1607,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.151"
version = "0.2.152"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"
[[package]]
name = "libgit2-sys"
@ -1678,9 +1672,9 @@ dependencies = [
[[package]]
name = "libz-sys"
version = "1.1.12"
version = "1.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b"
checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050"
dependencies = [
"cc",
"libc",
@ -2035,9 +2029,7 @@ name = "pep440_rs"
version = "0.3.12"
dependencies = [
"indoc",
"once_cell",
"pyo3",
"regex",
"serde",
"tracing",
"unicode-width",
@ -2179,7 +2171,7 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5699cc8a63d1aa2b1ee8e12b9ad70ac790d65788cd36101fa37f87ea46c4cef"
dependencies = [
"base64 0.21.5",
"base64 0.21.6",
"indexmap 2.1.0",
"line-wrap",
"quick-xml",
@ -2248,9 +2240,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.75"
version = "1.0.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "907a61bd0f64c2f29cd1cf1dc34d05176426a3f504a78010f08416ddb7b13708"
checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
dependencies = [
"unicode-ident",
]
@ -2540,12 +2532,14 @@ dependencies = [
name = "puffin-extract"
version = "0.0.1"
dependencies = [
"async_zip",
"flate2",
"fs-err",
"rayon",
"tar",
"thiserror",
"tokio",
"tokio-util",
"zip",
]
@ -2818,7 +2812,6 @@ dependencies = [
"pep440_rs 0.3.12",
"pep508_rs",
"puffin-normalize",
"puffin-warnings",
"regex",
"rfc2047-decoder",
"serde",
@ -3037,7 +3030,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41"
dependencies = [
"async-compression",
"base64 0.21.5",
"base64 0.21.6",
"bytes",
"encoding_rs",
"futures-core",
@ -3129,7 +3122,7 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e372613f15fc5171f9052b0c1fbafca5b1e5b0ba86aa13c9c39fd91ca1f7955"
dependencies = [
"base64 0.21.5",
"base64 0.21.6",
"charset",
"chumsky",
"memchr",
@ -3216,7 +3209,7 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
dependencies = [
"base64 0.21.5",
"base64 0.21.6",
]
[[package]]
@ -3886,9 +3879,9 @@ dependencies = [
[[package]]
name = "tracing-durations-export"
version = "0.1.0"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d6bb8898f56f636911130c78cc528338a2bb0426bdfb5a8fb523f98fc8da46d"
checksum = "b96372957860418808d5044039d88e6402e489b1d1f2a511a0dc201454268f73"
dependencies = [
"anyhow",
"fs-err",
@ -4319,15 +4312,6 @@ dependencies = [
"windows-targets 0.52.0",
]
[[package]]
name = "windows-sys"
version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
dependencies = [
"windows-targets 0.42.2",
]
[[package]]
name = "windows-sys"
version = "0.48.0"
@ -4346,21 +4330,6 @@ dependencies = [
"windows-targets 0.52.0",
]
[[package]]
name = "windows-targets"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
dependencies = [
"windows_aarch64_gnullvm 0.42.2",
"windows_aarch64_msvc 0.42.2",
"windows_i686_gnu 0.42.2",
"windows_i686_msvc 0.42.2",
"windows_x86_64_gnu 0.42.2",
"windows_x86_64_gnullvm 0.42.2",
"windows_x86_64_msvc 0.42.2",
]
[[package]]
name = "windows-targets"
version = "0.48.5"
@ -4391,12 +4360,6 @@ dependencies = [
"windows_x86_64_msvc 0.52.0",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.5"
@ -4409,12 +4372,6 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
[[package]]
name = "windows_aarch64_msvc"
version = "0.48.5"
@ -4427,12 +4384,6 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
[[package]]
name = "windows_i686_gnu"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
[[package]]
name = "windows_i686_gnu"
version = "0.48.5"
@ -4445,12 +4396,6 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
[[package]]
name = "windows_i686_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
[[package]]
name = "windows_i686_msvc"
version = "0.48.5"
@ -4463,12 +4408,6 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
[[package]]
name = "windows_x86_64_gnu"
version = "0.48.5"
@ -4481,12 +4420,6 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.48.5"
@ -4499,12 +4432,6 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
[[package]]
name = "windows_x86_64_msvc"
version = "0.48.5"
@ -4519,9 +4446,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
[[package]]
name = "winnow"
version = "0.5.32"
version = "0.5.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8434aeec7b290e8da5c3f0d628cb0eac6cabcb31d14bb74f779a08109a5914d6"
checksum = "b7cf47b659b318dccbd69cc4797a39ae128f533dce7902a1096044d1967b9c16"
dependencies = [
"memchr",
]

View File

@ -120,7 +120,7 @@ impl CachedDist {
}
impl CachedDirectUrlDist {
/// Initialize a [`CachedDirectUrlDist`] from a [`WheelFilename`], [`Url`], and [`Path`].
/// Initialize a [`CachedDirectUrlDist`] from a [`WheelFilename`], [`url::Url`], and [`Path`].
pub fn from_url(filename: WheelFilename, url: VerbatimUrl, path: PathBuf) -> Self {
Self {
filename,

View File

@ -35,8 +35,8 @@ serde_json = { workspace = true }
tempfile = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
tracing-subscriber = { workspace = true, optional = true }
which = { workspace = true }
[features]
cli = ["clap"]
cli = ["clap", "tracing-subscriber"]

View File

@ -17,9 +17,7 @@ name = "pep440_rs"
crate-type = ["rlib", "cdylib"]
[dependencies]
once_cell = { workspace = true }
pyo3 = { workspace = true, optional = true, features = ["extension-module", "abi3-py37"] }
regex = { workspace = true }
serde = { workspace = true, features = ["derive"], optional = true }
tracing = { workspace = true, optional = true }
unicode-width = { workspace = true }

View File

@ -12,10 +12,6 @@
//! assert!(version_specifiers.iter().all(|specifier| specifier.contains(&version)));
//! ```
//!
//! The error handling and diagnostics is a bit overdone because this my parser-and-diagnostics
//! learning project (which kinda failed because the byte based regex crate and char-based
//! diagnostics don't mix well)
//!
//! PEP 440 has a lot of unintuitive features, including:
//!
//! * An epoch that you can prefix the version which, e.g. `1!1.2.3`. Lower epoch always means lower

View File

@ -101,7 +101,6 @@ impl FromStr for Operator {
"<=" => Self::LessThanEqual,
">" => Self::GreaterThan,
">=" => Self::GreaterThanEqual,
// Should be forbidden by the regex if called from normal parsing
other => {
return Err(OperatorParseError {
got: other.to_string(),
@ -666,8 +665,7 @@ impl FromStr for Version {
/// Parses a version such as `1.19`, `1.0a1`,`1.0+abc.5` or `1!2012.2`
///
/// Note that this variant doesn't allow the version to end with a star, see
/// [`Self::from_str_star`] if you want to parse versions for specifiers
/// Note that this doesn't allow wildcard versions.
fn from_str(version: &str) -> Result<Self, Self::Err> {
Parser::new(version.as_bytes()).parse()
}
@ -2766,7 +2764,7 @@ mod tests {
}
#[test]
fn test_regex_mismatch() {
fn test_invalid_word() {
let result = Version::from_str("blergh");
assert_eq!(result.unwrap_err(), ErrorKind::NoLeadingNumber.into());
}

View File

@ -1279,7 +1279,7 @@ mod tests {
}
#[test]
fn test_regex_mismatch() {
fn test_invalid_word() {
let result = VersionSpecifiers::from_str("blergh");
assert_eq!(
result.unwrap_err().inner.err,

View File

@ -392,7 +392,7 @@ pub enum CacheBucket {
/// * `simple-v0/pypi/<package_name>.msgpack`
/// * `simple-v0/<digest(index_url)>/<package_name>.msgpack`
///
/// The response is parsed into [`puffin_client::SimpleMetadata`] before storage.
/// The response is parsed into `puffin_client::SimpleMetadata` before storage.
Simple,
}

View File

@ -151,7 +151,20 @@ pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> {
{
"Building source distributions is disabled".to_string()
} else {
format!("{err:?}")
err.chain()
.map(|err| {
let formatted = err.to_string();
// Cut overly long c/c++ compile output
if formatted.lines().count() > 20 {
let formatted: Vec<_> = formatted.lines().collect();
formatted[..20].join("\n")
+ "\n[...]\n"
+ &formatted[formatted.len() - 20..].join("\n")
} else {
formatted
}
})
.join("\n Caused by: ")
};
info!(
"Error for {} ({}/{}, {} ms): {}",

View File

@ -4,8 +4,8 @@ use std::path::Path;
use std::str::FromStr;
use std::sync::Arc;
use bytesize::ByteSize;
use fs_err::tokio as fs;
use puffin_extract::unzip_no_seek;
use thiserror::Error;
use tokio::task::JoinError;
use tokio_util::compat::FuturesAsyncReadCompatExt;
@ -21,12 +21,10 @@ use puffin_git::GitSource;
use puffin_traits::BuildContext;
use pypi_types::Metadata21;
use crate::download::BuiltWheel;
use crate::download::{BuiltWheel, UnzippedWheel};
use crate::locks::Locks;
use crate::reporter::Facade;
use crate::{
DiskWheel, InMemoryWheel, LocalWheel, Reporter, SourceDistCachedBuilder, SourceDistError,
};
use crate::{DiskWheel, LocalWheel, Reporter, SourceDistCachedBuilder, SourceDistError};
#[derive(Debug, Error)]
pub enum DistributionDatabaseError {
@ -37,6 +35,8 @@ pub enum DistributionDatabaseError {
#[error(transparent)]
Client(#[from] puffin_client::Error),
#[error(transparent)]
Extract(#[from] puffin_extract::Error),
#[error(transparent)]
Io(#[from] io::Error),
#[error(transparent)]
Distribution(#[from] distribution_types::Error),
@ -108,116 +108,68 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
) -> Result<LocalWheel, DistributionDatabaseError> {
match &dist {
Dist::Built(BuiltDist::Registry(wheel)) => {
// Fetch the wheel.
let url = wheel
.base
.join_relative(&wheel.file.url)
.map_err(|err| DistributionDatabaseError::Url(wheel.file.url.clone(), err))?;
// Make cache entry
let wheel_filename = WheelFilename::from_str(&wheel.file.filename)?;
let cache_entry = self.cache.entry(
CacheBucket::Wheels,
WheelCache::Index(&wheel.index).remote_wheel_dir(wheel_filename.name.as_ref()),
wheel_filename.stem(),
);
// Download and unzip on the same tokio task
//
// In all wheels we've seen so far, unzipping while downloading is
// faster than downloading into a file and then unzipping on multiple
// threads.
//
// Writing to a file first may be faster if the wheel takes longer to
// unzip than it takes to download. This may happen if the wheel is a
// zip bomb, or if the machine has a weak cpu (with many cores), but a
// fast network.
//
// If we find such a case, it may make sense to create separate tasks
// for downloading and unzipping (with a buffer in between) and switch
// to rayon if this buffer grows large by the time the file is fully
// downloaded.
let reader = self.client.stream_external(&url).await?;
let target = cache_entry.into_path_buf();
unzip_no_seek(reader.compat(), &target).await?;
// If the file is greater than 5MB, write it to disk; otherwise, keep it in memory.
let byte_size = wheel.file.size.map(ByteSize::b);
let local_wheel = if let Some(byte_size) =
byte_size.filter(|byte_size| *byte_size < ByteSize::mb(5))
{
debug!("Fetching in-memory wheel from registry: {dist} ({byte_size})",);
let cache_entry = self.cache.entry(
CacheBucket::Wheels,
WheelCache::Index(&wheel.index)
.remote_wheel_dir(wheel_filename.name.as_ref()),
wheel_filename.stem(),
);
// Read into a buffer.
let mut buffer = Vec::with_capacity(
wheel
.file
.size
.unwrap_or(0)
.try_into()
.expect("5MB shouldn't be bigger usize::MAX"),
);
let mut reader = tokio::io::BufReader::new(reader.compat());
tokio::io::copy(&mut reader, &mut buffer).await?;
LocalWheel::InMemory(InMemoryWheel {
dist: dist.clone(),
target: cache_entry.into_path_buf(),
buffer,
filename: wheel_filename,
})
} else {
let size =
byte_size.map_or("unknown size".to_string(), |size| size.to_string());
debug!("Fetching disk-based wheel from registry: {dist} ({size})");
let filename = wheel_filename.to_string();
// Download the wheel to a temporary file.
let temp_dir = tempfile::tempdir_in(self.cache.root())?;
let temp_file = temp_dir.path().join(&filename);
let mut writer =
tokio::io::BufWriter::new(tokio::fs::File::create(&temp_file).await?);
tokio::io::copy(&mut reader.compat(), &mut writer).await?;
// Move the temporary file to the cache.
let cache_entry = self.cache.entry(
CacheBucket::Wheels,
WheelCache::Index(&wheel.index)
.remote_wheel_dir(wheel_filename.name.as_ref()),
filename,
);
fs::create_dir_all(&cache_entry.dir()).await?;
tokio::fs::rename(temp_file, &cache_entry.path()).await?;
LocalWheel::Disk(DiskWheel {
dist: dist.clone(),
target: cache_entry
.with_file(wheel_filename.stem())
.path()
.to_path_buf(),
path: cache_entry.into_path_buf(),
filename: wheel_filename,
})
};
Ok(local_wheel)
Ok(LocalWheel::Unzipped(UnzippedWheel {
dist: dist.clone(),
target,
filename: wheel_filename,
}))
}
Dist::Built(BuiltDist::DirectUrl(wheel)) => {
debug!("Fetching disk-based wheel from URL: {}", wheel.url);
let reader = self.client.stream_external(&wheel.url).await?;
let filename = wheel.filename.to_string();
// Download the wheel to a temporary file.
// Download and unzip the wheel to a temporary dir.
let temp_dir = tempfile::tempdir_in(self.cache.root())?;
let temp_file = temp_dir.path().join(&filename);
let mut writer =
tokio::io::BufWriter::new(tokio::fs::File::create(&temp_file).await?);
tokio::io::copy(&mut reader.compat(), &mut writer).await?;
let temp_target = temp_dir.path().join(wheel.filename.to_string());
unzip_no_seek(reader.compat(), &temp_target).await?;
// Move the temporary file to the cache.
let cache_entry = self.cache.entry(
CacheBucket::Wheels,
WheelCache::Url(&wheel.url).remote_wheel_dir(wheel.name().as_ref()),
filename,
wheel.filename.stem(),
);
fs::create_dir_all(&cache_entry.dir()).await?;
tokio::fs::rename(temp_file, &cache_entry.path()).await?;
let target = cache_entry.into_path_buf();
tokio::fs::rename(temp_target, &target).await?;
let local_wheel = LocalWheel::Disk(DiskWheel {
let local_wheel = LocalWheel::Unzipped(UnzippedWheel {
dist: dist.clone(),
target: cache_entry
.with_file(wheel.filename.stem())
.path()
.to_path_buf(),
path: cache_entry.into_path_buf(),
target,
filename: wheel.filename.clone(),
});

View File

@ -10,16 +10,14 @@ use pypi_types::Metadata21;
use crate::error::Error;
/// A downloaded wheel that's stored in-memory.
/// A wheel that's been unzipped while downloading
#[derive(Debug, Clone)]
pub struct InMemoryWheel {
pub struct UnzippedWheel {
/// The remote distribution from which this wheel was downloaded.
pub(crate) dist: Dist,
/// The parsed filename.
pub(crate) filename: WheelFilename,
/// The contents of the wheel.
pub(crate) buffer: Vec<u8>,
/// The expected path to the downloaded wheel's entry in the cache.
/// The path in the cache dir where the wheel was downloaded.
pub(crate) target: PathBuf,
}
@ -52,7 +50,7 @@ pub struct BuiltWheel {
/// A downloaded or built wheel.
#[derive(Debug, Clone)]
pub enum LocalWheel {
InMemory(InMemoryWheel),
Unzipped(UnzippedWheel),
Disk(DiskWheel),
Built(BuiltWheel),
}
@ -61,7 +59,7 @@ impl LocalWheel {
/// Return the path to the downloaded wheel's entry in the cache.
pub fn target(&self) -> &Path {
match self {
LocalWheel::InMemory(wheel) => &wheel.target,
LocalWheel::Unzipped(wheel) => &wheel.target,
LocalWheel::Disk(wheel) => &wheel.target,
LocalWheel::Built(wheel) => &wheel.target,
}
@ -70,7 +68,7 @@ impl LocalWheel {
/// Return the [`Dist`] from which this wheel was downloaded.
pub fn remote(&self) -> &Dist {
match self {
LocalWheel::InMemory(wheel) => wheel.remote(),
LocalWheel::Unzipped(wheel) => wheel.remote(),
LocalWheel::Disk(wheel) => wheel.remote(),
LocalWheel::Built(wheel) => wheel.remote(),
}
@ -79,21 +77,21 @@ impl LocalWheel {
/// Return the [`WheelFilename`] of this wheel.
pub fn filename(&self) -> &WheelFilename {
match self {
LocalWheel::InMemory(wheel) => &wheel.filename,
LocalWheel::Unzipped(wheel) => &wheel.filename,
LocalWheel::Disk(wheel) => &wheel.filename,
LocalWheel::Built(wheel) => &wheel.filename,
}
}
}
impl DiskWheel {
impl UnzippedWheel {
/// Return the [`Dist`] from which this wheel was downloaded.
pub fn remote(&self) -> &Dist {
&self.dist
}
}
impl InMemoryWheel {
impl DiskWheel {
/// Return the [`Dist`] from which this wheel was downloaded.
pub fn remote(&self) -> &Dist {
&self.dist

View File

@ -1,5 +1,5 @@
pub use distribution_database::{DistributionDatabase, DistributionDatabaseError};
pub use download::{BuiltWheel, DiskWheel, InMemoryWheel, LocalWheel};
pub use download::{BuiltWheel, DiskWheel, LocalWheel};
pub use index::{BuiltWheelIndex, RegistryWheelIndex};
pub use reporter::Reporter;
pub use source::{SourceDistCachedBuilder, SourceDistError};

View File

@ -3,19 +3,13 @@ use std::path::Path;
use puffin_extract::{unzip_archive, Error};
use crate::download::BuiltWheel;
use crate::{DiskWheel, InMemoryWheel, LocalWheel};
use crate::{DiskWheel, LocalWheel};
pub trait Unzip {
/// Unzip a wheel into the target directory.
fn unzip(&self, target: &Path) -> Result<(), Error>;
}
impl Unzip for InMemoryWheel {
fn unzip(&self, target: &Path) -> Result<(), Error> {
unzip_archive(std::io::Cursor::new(&self.buffer), target)
}
}
impl Unzip for DiskWheel {
fn unzip(&self, target: &Path) -> Result<(), Error> {
unzip_archive(fs_err::File::open(&self.path)?, target)
@ -31,7 +25,7 @@ impl Unzip for BuiltWheel {
impl Unzip for LocalWheel {
fn unzip(&self, target: &Path) -> Result<(), Error> {
match self {
LocalWheel::InMemory(wheel) => wheel.unzip(target),
LocalWheel::Unzipped(_) => Ok(()),
LocalWheel::Disk(wheel) => wheel.unzip(target),
LocalWheel::Built(wheel) => wheel.unzip(target),
}

View File

@ -13,6 +13,8 @@ license = { workspace = true }
workspace = true
[dependencies]
tokio-util = { workspace = true, features = ["compat"] }
async_zip = { workspace = true, features = ["tokio"] }
flate2 = { workspace = true }
fs-err = { workspace = true }
rayon = { workspace = true }

View File

@ -1,6 +1,7 @@
use std::path::{Path, PathBuf};
use rayon::prelude::*;
use tokio_util::compat::FuturesAsyncReadCompatExt;
use zip::result::ZipError;
use zip::ZipArchive;
@ -13,6 +14,8 @@ pub enum Error {
#[error(transparent)]
Zip(#[from] ZipError),
#[error(transparent)]
AsyncZip(#[from] async_zip::error::ZipError),
#[error(transparent)]
Io(#[from] std::io::Error),
#[error("Unsupported archive type: {0}")]
UnsupportedArchive(PathBuf),
@ -22,6 +25,44 @@ pub enum Error {
InvalidArchive(Vec<fs_err::DirEntry>),
}
/// Unzip a `.zip` archive into the target directory without requiring Seek.
///
/// This is useful for unzipping files as they're being downloaded. If the archive
/// is already fully on disk, consider using `unzip_archive`, which can use multiple
/// threads to work faster in that case.
pub async fn unzip_no_seek<R: tokio::io::AsyncRead + Unpin>(
reader: R,
target: &Path,
) -> Result<(), Error> {
let mut zip = async_zip::base::read::stream::ZipFileReader::with_tokio(reader);
while let Some(mut entry) = zip.next_with_entry().await? {
// Construct path
let path = entry.reader().entry().filename().as_str()?;
let path = target.join(path);
let is_dir = entry.reader().entry().dir()?;
// Create dir or write file
if is_dir {
tokio::fs::create_dir_all(path).await?;
} else {
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent).await?;
}
let file = tokio::fs::File::create(path).await?;
let mut writer = tokio::io::BufWriter::new(file);
let mut reader = entry.reader_mut().compat();
tokio::io::copy(&mut reader, &mut writer).await?;
}
// Close current file to get access to the next one. See docs:
// https://docs.rs/async_zip/0.0.16/async_zip/base/read/stream/
zip = entry.skip().await?;
}
Ok(())
}
/// Unzip a `.zip` archive into the target directory.
pub fn unzip_archive<R: Send + std::io::Read + std::io::Seek + HasLength>(
reader: R,

View File

@ -205,33 +205,39 @@ impl<'a, Context: BuildContext + Send + Sync> Downloader<'a, Context> {
}
// Unzip the wheel.
let normalized_path = tokio::task::spawn_blocking({
move || -> Result<PathBuf, puffin_extract::Error> {
// Unzip the wheel into a temporary directory.
let parent = download
.target()
.parent()
.expect("Cache paths can't be root");
fs_err::create_dir_all(parent)?;
let staging = tempfile::tempdir_in(parent)?;
download.unzip(staging.path())?;
let normalized_path = if matches!(download, LocalWheel::Unzipped(_)) {
// Just an optimizaion: Avoid spawning a blocking
// task if there is no work to be done.
download.target().to_path_buf()
} else {
tokio::task::spawn_blocking({
move || -> Result<PathBuf, puffin_extract::Error> {
// Unzip the wheel into a temporary directory.
let parent = download
.target()
.parent()
.expect("Cache paths can't be root");
fs_err::create_dir_all(parent)?;
let staging = tempfile::tempdir_in(parent)?;
download.unzip(staging.path())?;
// Move the unzipped wheel into the cache.
if let Err(err) = fs_err::rename(staging.into_path(), download.target()) {
// If another thread already unpacked the wheel, we can ignore the error.
return if download.target().is_dir() {
warn!("Wheel is already unpacked: {}", download.remote());
Ok(download.target().to_path_buf())
} else {
Err(err.into())
};
// Move the unzipped wheel into the cache.
if let Err(err) = fs_err::rename(staging.into_path(), download.target()) {
// If another thread already unpacked the wheel, we can ignore the error.
return if download.target().is_dir() {
warn!("Wheel is already unpacked: {}", download.remote());
Ok(download.target().to_path_buf())
} else {
Err(err.into())
};
}
Ok(download.target().to_path_buf())
}
Ok(download.target().to_path_buf())
}
})
.await?
.map_err(|err| Error::Unzip(remote.clone(), err))?;
})
.await?
.map_err(|err| Error::Unzip(remote.clone(), err))?
};
Ok(CachedDist::from_remote(remote, filename, normalized_path))
}

View File

@ -4,6 +4,7 @@ use std::time::SystemTimeError;
use thiserror::Error;
pub use crate::cfg::Configuration;
pub use crate::interpreter::Interpreter;
pub use crate::python_version::PythonVersion;
pub use crate::virtual_env::Virtualenv;

View File

@ -2,7 +2,7 @@ pub use error::ResolveError;
pub use finder::{DistFinder, Reporter as FinderReporter};
pub use manifest::Manifest;
pub use prerelease_mode::PreReleaseMode;
pub use resolution::ResolutionGraph;
pub use resolution::{Diagnostic, ResolutionGraph};
pub use resolution_mode::ResolutionMode;
pub use resolution_options::ResolutionOptions;
pub use resolver::{BuildId, Reporter as ResolverReporter, Resolver, ResolverProvider};

View File

@ -37,7 +37,7 @@ pub trait ResolverProvider: Send + Sync {
dist: &'io Dist,
) -> impl Future<Output = WheelMetadataResponse> + Send + 'io;
/// Set the [`Reporter`] to use for this installer.
/// Set the [`puffin_distribution::Reporter`] to use for this installer.
#[must_use]
fn with_reporter(self, reporter: impl puffin_distribution::Reporter + 'static) -> Self;
}

View File

@ -16,7 +16,6 @@ workspace = true
pep440_rs = { path = "../pep440-rs", features = ["serde"] }
pep508_rs = { path = "../pep508-rs", features = ["serde"] }
puffin-normalize = { path = "../puffin-normalize" }
puffin-warnings = { path = "../puffin-warnings" }
chrono = { workspace = true, features = ["serde"] }
mailparse = { workspace = true }