From 8c2b7d55afabc9348fcd6fcda1d9338044698a74 Mon Sep 17 00:00:00 2001 From: konsti Date: Thu, 11 Jan 2024 11:43:40 +0100 Subject: [PATCH 1/6] Cleanup deps and docs (#882) Fix warnings from `cargo +nightly udeps` and `cargo doc`. Removes all mentions of regex from pep440_rs. --- Cargo.lock | 3 --- crates/distribution-types/src/cached.rs | 2 +- crates/gourgeist/Cargo.toml | 4 ++-- crates/pep440-rs/Cargo.toml | 2 -- crates/pep440-rs/src/lib.rs | 4 ---- crates/pep440-rs/src/version.rs | 6 ++---- crates/pep440-rs/src/version_specifier.rs | 2 +- crates/puffin-cache/src/lib.rs | 2 +- crates/puffin-interpreter/src/lib.rs | 1 + crates/puffin-resolver/src/lib.rs | 2 +- crates/puffin-resolver/src/resolver/provider.rs | 2 +- crates/pypi-types/Cargo.toml | 1 - 12 files changed, 10 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a37c8d94..c31020db5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2035,9 +2035,7 @@ name = "pep440_rs" version = "0.3.12" dependencies = [ "indoc", - "once_cell", "pyo3", - "regex", "serde", "tracing", "unicode-width", @@ -2818,7 +2816,6 @@ dependencies = [ "pep440_rs 0.3.12", "pep508_rs", "puffin-normalize", - "puffin-warnings", "regex", "rfc2047-decoder", "serde", diff --git a/crates/distribution-types/src/cached.rs b/crates/distribution-types/src/cached.rs index a37d510d6..b1c08475f 100644 --- a/crates/distribution-types/src/cached.rs +++ b/crates/distribution-types/src/cached.rs @@ -120,7 +120,7 @@ impl CachedDist { } impl CachedDirectUrlDist { - /// Initialize a [`CachedDirectUrlDist`] from a [`WheelFilename`], [`Url`], and [`Path`]. + /// Initialize a [`CachedDirectUrlDist`] from a [`WheelFilename`], [`url::Url`], and [`Path`]. pub fn from_url(filename: WheelFilename, url: VerbatimUrl, path: PathBuf) -> Self { Self { filename, diff --git a/crates/gourgeist/Cargo.toml b/crates/gourgeist/Cargo.toml index 395ee705e..c76ab0458 100644 --- a/crates/gourgeist/Cargo.toml +++ b/crates/gourgeist/Cargo.toml @@ -35,8 +35,8 @@ serde_json = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } +tracing-subscriber = { workspace = true, optional = true } which = { workspace = true } [features] -cli = ["clap"] +cli = ["clap", "tracing-subscriber"] diff --git a/crates/pep440-rs/Cargo.toml b/crates/pep440-rs/Cargo.toml index faa3eedcd..c2f0682d2 100644 --- a/crates/pep440-rs/Cargo.toml +++ b/crates/pep440-rs/Cargo.toml @@ -17,9 +17,7 @@ name = "pep440_rs" crate-type = ["rlib", "cdylib"] [dependencies] -once_cell = { workspace = true } pyo3 = { workspace = true, optional = true, features = ["extension-module", "abi3-py37"] } -regex = { workspace = true } serde = { workspace = true, features = ["derive"], optional = true } tracing = { workspace = true, optional = true } unicode-width = { workspace = true } diff --git a/crates/pep440-rs/src/lib.rs b/crates/pep440-rs/src/lib.rs index 94b3eccb8..0dec2f9dd 100644 --- a/crates/pep440-rs/src/lib.rs +++ b/crates/pep440-rs/src/lib.rs @@ -12,10 +12,6 @@ //! assert!(version_specifiers.iter().all(|specifier| specifier.contains(&version))); //! ``` //! -//! The error handling and diagnostics is a bit overdone because this my parser-and-diagnostics -//! learning project (which kinda failed because the byte based regex crate and char-based -//! diagnostics don't mix well) -//! //! PEP 440 has a lot of unintuitive features, including: //! //! * An epoch that you can prefix the version which, e.g. `1!1.2.3`. Lower epoch always means lower diff --git a/crates/pep440-rs/src/version.rs b/crates/pep440-rs/src/version.rs index ce26dce01..d29724e65 100644 --- a/crates/pep440-rs/src/version.rs +++ b/crates/pep440-rs/src/version.rs @@ -101,7 +101,6 @@ impl FromStr for Operator { "<=" => Self::LessThanEqual, ">" => Self::GreaterThan, ">=" => Self::GreaterThanEqual, - // Should be forbidden by the regex if called from normal parsing other => { return Err(OperatorParseError { got: other.to_string(), @@ -666,8 +665,7 @@ impl FromStr for Version { /// Parses a version such as `1.19`, `1.0a1`,`1.0+abc.5` or `1!2012.2` /// - /// Note that this variant doesn't allow the version to end with a star, see - /// [`Self::from_str_star`] if you want to parse versions for specifiers + /// Note that this doesn't allow wildcard versions. fn from_str(version: &str) -> Result { Parser::new(version.as_bytes()).parse() } @@ -2766,7 +2764,7 @@ mod tests { } #[test] - fn test_regex_mismatch() { + fn test_invalid_word() { let result = Version::from_str("blergh"); assert_eq!(result.unwrap_err(), ErrorKind::NoLeadingNumber.into()); } diff --git a/crates/pep440-rs/src/version_specifier.rs b/crates/pep440-rs/src/version_specifier.rs index 69040735f..1e54d00e9 100644 --- a/crates/pep440-rs/src/version_specifier.rs +++ b/crates/pep440-rs/src/version_specifier.rs @@ -1279,7 +1279,7 @@ mod tests { } #[test] - fn test_regex_mismatch() { + fn test_invalid_word() { let result = VersionSpecifiers::from_str("blergh"); assert_eq!( result.unwrap_err().inner.err, diff --git a/crates/puffin-cache/src/lib.rs b/crates/puffin-cache/src/lib.rs index 12a9b2e89..ba20e08df 100644 --- a/crates/puffin-cache/src/lib.rs +++ b/crates/puffin-cache/src/lib.rs @@ -392,7 +392,7 @@ pub enum CacheBucket { /// * `simple-v0/pypi/.msgpack` /// * `simple-v0//.msgpack` /// - /// The response is parsed into [`puffin_client::SimpleMetadata`] before storage. + /// The response is parsed into `puffin_client::SimpleMetadata` before storage. Simple, } diff --git a/crates/puffin-interpreter/src/lib.rs b/crates/puffin-interpreter/src/lib.rs index cf060f830..8bbd5b2fc 100644 --- a/crates/puffin-interpreter/src/lib.rs +++ b/crates/puffin-interpreter/src/lib.rs @@ -4,6 +4,7 @@ use std::time::SystemTimeError; use thiserror::Error; +pub use crate::cfg::Configuration; pub use crate::interpreter::Interpreter; pub use crate::python_version::PythonVersion; pub use crate::virtual_env::Virtualenv; diff --git a/crates/puffin-resolver/src/lib.rs b/crates/puffin-resolver/src/lib.rs index fba3709bc..bcedbd454 100644 --- a/crates/puffin-resolver/src/lib.rs +++ b/crates/puffin-resolver/src/lib.rs @@ -2,7 +2,7 @@ pub use error::ResolveError; pub use finder::{DistFinder, Reporter as FinderReporter}; pub use manifest::Manifest; pub use prerelease_mode::PreReleaseMode; -pub use resolution::ResolutionGraph; +pub use resolution::{Diagnostic, ResolutionGraph}; pub use resolution_mode::ResolutionMode; pub use resolution_options::ResolutionOptions; pub use resolver::{BuildId, Reporter as ResolverReporter, Resolver, ResolverProvider}; diff --git a/crates/puffin-resolver/src/resolver/provider.rs b/crates/puffin-resolver/src/resolver/provider.rs index a83ea03ec..ae0dd3e09 100644 --- a/crates/puffin-resolver/src/resolver/provider.rs +++ b/crates/puffin-resolver/src/resolver/provider.rs @@ -37,7 +37,7 @@ pub trait ResolverProvider: Send + Sync { dist: &'io Dist, ) -> impl Future + Send + 'io; - /// Set the [`Reporter`] to use for this installer. + /// Set the [`puffin_distribution::Reporter`] to use for this installer. #[must_use] fn with_reporter(self, reporter: impl puffin_distribution::Reporter + 'static) -> Self; } diff --git a/crates/pypi-types/Cargo.toml b/crates/pypi-types/Cargo.toml index 5ed1a4b58..c998935b4 100644 --- a/crates/pypi-types/Cargo.toml +++ b/crates/pypi-types/Cargo.toml @@ -16,7 +16,6 @@ workspace = true pep440_rs = { path = "../pep440-rs", features = ["serde"] } pep508_rs = { path = "../pep508-rs", features = ["serde"] } puffin-normalize = { path = "../puffin-normalize" } -puffin-warnings = { path = "../puffin-warnings" } chrono = { workspace = true, features = ["serde"] } mailparse = { workspace = true } From 0dfbddd27545e0664aa4bb312b9d73916c4e48ab Mon Sep 17 00:00:00 2001 From: konsti Date: Thu, 11 Jan 2024 14:53:13 +0100 Subject: [PATCH 2/6] Shorten resolve many dev output (#885) --- crates/puffin-dev/src/resolve_many.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/crates/puffin-dev/src/resolve_many.rs b/crates/puffin-dev/src/resolve_many.rs index 9b3d5f3e4..1dc73fcd1 100644 --- a/crates/puffin-dev/src/resolve_many.rs +++ b/crates/puffin-dev/src/resolve_many.rs @@ -151,7 +151,20 @@ pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> { { "Building source distributions is disabled".to_string() } else { - format!("{err:?}") + err.chain() + .map(|err| { + let formatted = err.to_string(); + // Cut overly long c/c++ compile output + if formatted.lines().count() > 20 { + let formatted: Vec<_> = formatted.lines().collect(); + formatted[..20].join("\n") + + "\n[...]\n" + + &formatted[formatted.len() - 20..].join("\n") + } else { + formatted + } + }) + .join("\n Caused by: ") }; info!( "Error for {} ({}/{}, {} ms): {}", From 4123a35228a46bae526272d1491bb56a644da191 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 11 Jan 2024 09:10:07 -0500 Subject: [PATCH 3/6] Run `cargo update` (#873) --- Cargo.lock | 142 +++++++++++++---------------------------------------- 1 file changed, 35 insertions(+), 107 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c31020db5..0d403b3e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -282,9 +282,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.5" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "c79fed4cdb43e993fcdadc7e58a09fd0e3e649c4436fa11da71c9f1f3ee7feb9" [[package]] name = "bench" @@ -519,9 +519,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.13" +version = "4.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bdc885e4cacc7f7c9eedc1ef6da641603180c783c41a15c264944deeaab642" +checksum = "33e92c5c1a78c62968ec57dbc2440366a2d6e5a23faf829970ff1585dc6b18e2" dependencies = [ "clap_builder", "clap_derive", @@ -529,9 +529,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.12" +version = "4.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" +checksum = "f4323769dc8a61e2c39ad7dc26f6f2800524691a44d74fe3d1071a5c24db6370" dependencies = [ "anstream", "anstyle", @@ -580,15 +580,15 @@ checksum = "4ec6d3da8e550377a85339063af6e3735f4b1d9392108da4e083a1b3b9820288" [[package]] name = "console" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] @@ -681,34 +681,28 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if 1.0.0", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.17" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if 1.0.0", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -1029,9 +1023,9 @@ checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeee267a1883f7ebef3700f262d2d54de95dfaf38189015a74fdc4e0c7ad8143" +checksum = "445ba825b27408685aaecefd65178908c36c6e96aaf6d8599419d46e624192ba" dependencies = [ "fastrand", "futures-core", @@ -1093,9 +1087,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -1399,9 +1393,9 @@ dependencies = [ [[package]] name = "ignore" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747ad1b4ae841a78e8aba0d63adbfbeaea26b517b63705d47856b73015d27060" +checksum = "b46810df39e66e925525d6e38ce1e7f6e1d208f72dc39757880fcb66e2c58af1" dependencies = [ "crossbeam-deque", "globset", @@ -1613,9 +1607,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.151" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libgit2-sys" @@ -1678,9 +1672,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050" dependencies = [ "cc", "libc", @@ -2177,7 +2171,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5699cc8a63d1aa2b1ee8e12b9ad70ac790d65788cd36101fa37f87ea46c4cef" dependencies = [ - "base64 0.21.5", + "base64 0.21.6", "indexmap 2.1.0", "line-wrap", "quick-xml", @@ -2246,9 +2240,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.75" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907a61bd0f64c2f29cd1cf1dc34d05176426a3f504a78010f08416ddb7b13708" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -3034,7 +3028,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "async-compression", - "base64 0.21.5", + "base64 0.21.6", "bytes", "encoding_rs", "futures-core", @@ -3126,7 +3120,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e372613f15fc5171f9052b0c1fbafca5b1e5b0ba86aa13c9c39fd91ca1f7955" dependencies = [ - "base64 0.21.5", + "base64 0.21.6", "charset", "chumsky", "memchr", @@ -3213,7 +3207,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.5", + "base64 0.21.6", ] [[package]] @@ -3883,9 +3877,9 @@ dependencies = [ [[package]] name = "tracing-durations-export" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d6bb8898f56f636911130c78cc528338a2bb0426bdfb5a8fb523f98fc8da46d" +checksum = "b96372957860418808d5044039d88e6402e489b1d1f2a511a0dc201454268f73" dependencies = [ "anyhow", "fs-err", @@ -4316,15 +4310,6 @@ dependencies = [ "windows-targets 0.52.0", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -4343,21 +4328,6 @@ dependencies = [ "windows-targets 0.52.0", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -4388,12 +4358,6 @@ dependencies = [ "windows_x86_64_msvc 0.52.0", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -4406,12 +4370,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -4424,12 +4382,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -4442,12 +4394,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -4460,12 +4406,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -4478,12 +4418,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -4496,12 +4430,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -4516,9 +4444,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.32" +version = "0.5.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8434aeec7b290e8da5c3f0d628cb0eac6cabcb31d14bb74f779a08109a5914d6" +checksum = "b7cf47b659b318dccbd69cc4797a39ae128f533dce7902a1096044d1967b9c16" dependencies = [ "memchr", ] From 10227a74f889e1345240887e860ad0ec87d747ae Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Thu, 11 Jan 2024 09:41:46 -0500 Subject: [PATCH 4/6] Unzip while downloading (#856) --- Cargo.lock | 6 +- .../src/distribution_database.rs | 77 +++++++++++++------ crates/puffin-distribution/src/download.rs | 22 ++++++ crates/puffin-distribution/src/unzip.rs | 1 + crates/puffin-extract/Cargo.toml | 2 + crates/puffin-extract/src/lib.rs | 41 ++++++++++ crates/puffin-installer/src/downloader.rs | 56 ++++++++------ 7 files changed, 154 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d403b3e2..919e9a240 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1183,9 +1183,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "b553656127a00601c8ae5590fcfdc118e4083a7924b6cf4ffc1ea4b99dc429d7" dependencies = [ "bytes", "fnv", @@ -2532,12 +2532,14 @@ dependencies = [ name = "puffin-extract" version = "0.0.1" dependencies = [ + "async_zip", "flate2", "fs-err", "rayon", "tar", "thiserror", "tokio", + "tokio-util", "zip", ] diff --git a/crates/puffin-distribution/src/distribution_database.rs b/crates/puffin-distribution/src/distribution_database.rs index 08aecdbd8..3baa5a9cc 100644 --- a/crates/puffin-distribution/src/distribution_database.rs +++ b/crates/puffin-distribution/src/distribution_database.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use bytesize::ByteSize; use fs_err::tokio as fs; +use puffin_extract::unzip_no_seek; use thiserror::Error; use tokio::task::JoinError; use tokio_util::compat::FuturesAsyncReadCompatExt; @@ -21,7 +22,7 @@ use puffin_git::GitSource; use puffin_traits::BuildContext; use pypi_types::Metadata21; -use crate::download::BuiltWheel; +use crate::download::{BuiltWheel, UnzippedWheel}; use crate::locks::Locks; use crate::reporter::Facade; use crate::{ @@ -37,6 +38,8 @@ pub enum DistributionDatabaseError { #[error(transparent)] Client(#[from] puffin_client::Error), #[error(transparent)] + Extract(#[from] puffin_extract::Error), + #[error(transparent)] Io(#[from] io::Error), #[error(transparent)] Distribution(#[from] distribution_types::Error), @@ -108,30 +111,62 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context> ) -> Result { match &dist { Dist::Built(BuiltDist::Registry(wheel)) => { - // Fetch the wheel. let url = wheel .base .join_relative(&wheel.file.url) .map_err(|err| DistributionDatabaseError::Url(wheel.file.url.clone(), err))?; + // Make cache entry let wheel_filename = WheelFilename::from_str(&wheel.file.filename)?; + let cache_entry = self.cache.entry( + CacheBucket::Wheels, + WheelCache::Index(&wheel.index).remote_wheel_dir(wheel_filename.name.as_ref()), + wheel_filename.stem(), + ); + // Start the download let reader = self.client.stream_external(&url).await?; + // In all wheels we've seen so far, unzipping while downloading is the + // faster option. + // + // Writing to a file first may be faster if the wheel takes longer to + // unzip than it takes to download. This may happen if the wheel is a + // zip bomb, or if the machine has a weak cpu (with many cores), but a + // fast network. + // + // If we find such a case, it may make sense to create separate tasks + // for downloading and unzipping (with a buffer in between) and switch + // to rayon if this buffer grows large by the time the file is fully + // downloaded. + let unzip_while_downloading = true; + if unzip_while_downloading { + // Download and unzip to a temporary dir + let temp_dir = tempfile::tempdir_in(self.cache.root())?; + let temp_target = temp_dir.path().join(&wheel.file.filename); + unzip_no_seek(reader.compat(), &temp_target).await?; + + // Move the dir to the right place + fs::create_dir_all(&cache_entry.dir()).await?; + let target = cache_entry.into_path_buf(); + tokio::fs::rename(temp_target, &target).await?; + + return Ok(LocalWheel::Unzipped(UnzippedWheel { + dist: dist.clone(), + target, + filename: wheel_filename, + })); + } + // If the file is greater than 5MB, write it to disk; otherwise, keep it in memory. + // + // TODO this is currently dead code. Consider deleting if there's no use for it. let byte_size = wheel.file.size.map(ByteSize::b); let local_wheel = if let Some(byte_size) = byte_size.filter(|byte_size| *byte_size < ByteSize::mb(5)) { debug!("Fetching in-memory wheel from registry: {dist} ({byte_size})",); - let cache_entry = self.cache.entry( - CacheBucket::Wheels, - WheelCache::Index(&wheel.index) - .remote_wheel_dir(wheel_filename.name.as_ref()), - wheel_filename.stem(), - ); - // Read into a buffer. let mut buffer = Vec::with_capacity( wheel @@ -170,7 +205,7 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context> CacheBucket::Wheels, WheelCache::Index(&wheel.index) .remote_wheel_dir(wheel_filename.name.as_ref()), - filename, + filename, // TODO should this be filename.stem() to match the other branch? ); fs::create_dir_all(&cache_entry.dir()).await?; tokio::fs::rename(temp_file, &cache_entry.path()).await?; @@ -193,31 +228,25 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context> debug!("Fetching disk-based wheel from URL: {}", wheel.url); let reader = self.client.stream_external(&wheel.url).await?; - let filename = wheel.filename.to_string(); - // Download the wheel to a temporary file. + // Download and unzip the wheel to a temporary dir. let temp_dir = tempfile::tempdir_in(self.cache.root())?; - let temp_file = temp_dir.path().join(&filename); - let mut writer = - tokio::io::BufWriter::new(tokio::fs::File::create(&temp_file).await?); - tokio::io::copy(&mut reader.compat(), &mut writer).await?; + let temp_target = temp_dir.path().join(wheel.filename.to_string()); + unzip_no_seek(reader.compat(), &temp_target).await?; // Move the temporary file to the cache. let cache_entry = self.cache.entry( CacheBucket::Wheels, WheelCache::Url(&wheel.url).remote_wheel_dir(wheel.name().as_ref()), - filename, + wheel.filename.stem(), ); fs::create_dir_all(&cache_entry.dir()).await?; - tokio::fs::rename(temp_file, &cache_entry.path()).await?; + let target = cache_entry.into_path_buf(); + tokio::fs::rename(temp_target, &target).await?; - let local_wheel = LocalWheel::Disk(DiskWheel { + let local_wheel = LocalWheel::Unzipped(UnzippedWheel { dist: dist.clone(), - target: cache_entry - .with_file(wheel.filename.stem()) - .path() - .to_path_buf(), - path: cache_entry.into_path_buf(), + target, filename: wheel.filename.clone(), }); diff --git a/crates/puffin-distribution/src/download.rs b/crates/puffin-distribution/src/download.rs index 47dff95bf..b3ca4a98d 100644 --- a/crates/puffin-distribution/src/download.rs +++ b/crates/puffin-distribution/src/download.rs @@ -10,6 +10,17 @@ use pypi_types::Metadata21; use crate::error::Error; +/// A wheel that's been unzipped while downloading +#[derive(Debug, Clone)] +pub struct UnzippedWheel { + /// The remote distribution from which this wheel was downloaded. + pub(crate) dist: Dist, + /// The parsed filename. + pub(crate) filename: WheelFilename, + /// The path in the cache dir where the wheel was downloaded. + pub(crate) target: PathBuf, +} + /// A downloaded wheel that's stored in-memory. #[derive(Debug, Clone)] pub struct InMemoryWheel { @@ -52,6 +63,7 @@ pub struct BuiltWheel { /// A downloaded or built wheel. #[derive(Debug, Clone)] pub enum LocalWheel { + Unzipped(UnzippedWheel), InMemory(InMemoryWheel), Disk(DiskWheel), Built(BuiltWheel), @@ -61,6 +73,7 @@ impl LocalWheel { /// Return the path to the downloaded wheel's entry in the cache. pub fn target(&self) -> &Path { match self { + LocalWheel::Unzipped(wheel) => &wheel.target, LocalWheel::InMemory(wheel) => &wheel.target, LocalWheel::Disk(wheel) => &wheel.target, LocalWheel::Built(wheel) => &wheel.target, @@ -70,6 +83,7 @@ impl LocalWheel { /// Return the [`Dist`] from which this wheel was downloaded. pub fn remote(&self) -> &Dist { match self { + LocalWheel::Unzipped(wheel) => wheel.remote(), LocalWheel::InMemory(wheel) => wheel.remote(), LocalWheel::Disk(wheel) => wheel.remote(), LocalWheel::Built(wheel) => wheel.remote(), @@ -79,6 +93,7 @@ impl LocalWheel { /// Return the [`WheelFilename`] of this wheel. pub fn filename(&self) -> &WheelFilename { match self { + LocalWheel::Unzipped(wheel) => &wheel.filename, LocalWheel::InMemory(wheel) => &wheel.filename, LocalWheel::Disk(wheel) => &wheel.filename, LocalWheel::Built(wheel) => &wheel.filename, @@ -86,6 +101,13 @@ impl LocalWheel { } } +impl UnzippedWheel { + /// Return the [`Dist`] from which this wheel was downloaded. + pub fn remote(&self) -> &Dist { + &self.dist + } +} + impl DiskWheel { /// Return the [`Dist`] from which this wheel was downloaded. pub fn remote(&self) -> &Dist { diff --git a/crates/puffin-distribution/src/unzip.rs b/crates/puffin-distribution/src/unzip.rs index d7192605b..6708cc576 100644 --- a/crates/puffin-distribution/src/unzip.rs +++ b/crates/puffin-distribution/src/unzip.rs @@ -31,6 +31,7 @@ impl Unzip for BuiltWheel { impl Unzip for LocalWheel { fn unzip(&self, target: &Path) -> Result<(), Error> { match self { + LocalWheel::Unzipped(_) => Ok(()), LocalWheel::InMemory(wheel) => wheel.unzip(target), LocalWheel::Disk(wheel) => wheel.unzip(target), LocalWheel::Built(wheel) => wheel.unzip(target), diff --git a/crates/puffin-extract/Cargo.toml b/crates/puffin-extract/Cargo.toml index d34889118..86eb5a110 100644 --- a/crates/puffin-extract/Cargo.toml +++ b/crates/puffin-extract/Cargo.toml @@ -13,6 +13,8 @@ license = { workspace = true } workspace = true [dependencies] +tokio-util = { workspace = true, features = ["compat"] } +async_zip = { workspace = true, features = ["tokio"] } flate2 = { workspace = true } fs-err = { workspace = true } rayon = { workspace = true } diff --git a/crates/puffin-extract/src/lib.rs b/crates/puffin-extract/src/lib.rs index 02cccdd92..4a40b5b69 100644 --- a/crates/puffin-extract/src/lib.rs +++ b/crates/puffin-extract/src/lib.rs @@ -1,6 +1,7 @@ use std::path::{Path, PathBuf}; use rayon::prelude::*; +use tokio_util::compat::FuturesAsyncReadCompatExt; use zip::result::ZipError; use zip::ZipArchive; @@ -13,6 +14,8 @@ pub enum Error { #[error(transparent)] Zip(#[from] ZipError), #[error(transparent)] + AsyncZip(#[from] async_zip::error::ZipError), + #[error(transparent)] Io(#[from] std::io::Error), #[error("Unsupported archive type: {0}")] UnsupportedArchive(PathBuf), @@ -22,6 +25,44 @@ pub enum Error { InvalidArchive(Vec), } +/// Unzip a `.zip` archive into the target directory without requiring Seek. +/// +/// This is useful for unzipping files as they're being downloaded. If the archive +/// is already fully on disk, consider using `unzip_archive`, which can use multiple +/// threads to work faster in that case. +pub async fn unzip_no_seek( + reader: R, + target: &Path, +) -> Result<(), Error> { + let mut zip = async_zip::base::read::stream::ZipFileReader::with_tokio(reader); + + while let Some(mut entry) = zip.next_with_entry().await? { + // Construct path + let path = entry.reader().entry().filename().as_str()?; + let path = target.join(path); + let is_dir = entry.reader().entry().dir()?; + + // Create dir or write file + if is_dir { + tokio::fs::create_dir_all(path).await?; + } else { + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + let file = tokio::fs::File::create(path).await?; + let mut writer = tokio::io::BufWriter::new(file); + let mut reader = entry.reader_mut().compat(); + tokio::io::copy(&mut reader, &mut writer).await?; + } + + // Close current file to get access to the next one. See docs: + // https://docs.rs/async_zip/0.0.16/async_zip/base/read/stream/ + zip = entry.skip().await?; + } + + Ok(()) +} + /// Unzip a `.zip` archive into the target directory. pub fn unzip_archive( reader: R, diff --git a/crates/puffin-installer/src/downloader.rs b/crates/puffin-installer/src/downloader.rs index 0dfbcc0cd..9e36bef71 100644 --- a/crates/puffin-installer/src/downloader.rs +++ b/crates/puffin-installer/src/downloader.rs @@ -205,33 +205,39 @@ impl<'a, Context: BuildContext + Send + Sync> Downloader<'a, Context> { } // Unzip the wheel. - let normalized_path = tokio::task::spawn_blocking({ - move || -> Result { - // Unzip the wheel into a temporary directory. - let parent = download - .target() - .parent() - .expect("Cache paths can't be root"); - fs_err::create_dir_all(parent)?; - let staging = tempfile::tempdir_in(parent)?; - download.unzip(staging.path())?; + let normalized_path = if matches!(download, LocalWheel::Unzipped(_)) { + // Just an optimizaion: Avoid spawning a blocking + // task if there is no work to be done. + download.target().to_path_buf() + } else { + tokio::task::spawn_blocking({ + move || -> Result { + // Unzip the wheel into a temporary directory. + let parent = download + .target() + .parent() + .expect("Cache paths can't be root"); + fs_err::create_dir_all(parent)?; + let staging = tempfile::tempdir_in(parent)?; + download.unzip(staging.path())?; - // Move the unzipped wheel into the cache. - if let Err(err) = fs_err::rename(staging.into_path(), download.target()) { - // If another thread already unpacked the wheel, we can ignore the error. - return if download.target().is_dir() { - warn!("Wheel is already unpacked: {}", download.remote()); - Ok(download.target().to_path_buf()) - } else { - Err(err.into()) - }; + // Move the unzipped wheel into the cache. + if let Err(err) = fs_err::rename(staging.into_path(), download.target()) { + // If another thread already unpacked the wheel, we can ignore the error. + return if download.target().is_dir() { + warn!("Wheel is already unpacked: {}", download.remote()); + Ok(download.target().to_path_buf()) + } else { + Err(err.into()) + }; + } + + Ok(download.target().to_path_buf()) } - - Ok(download.target().to_path_buf()) - } - }) - .await? - .map_err(|err| Error::Unzip(remote.clone(), err))?; + }) + .await? + .map_err(|err| Error::Unzip(remote.clone(), err))? + }; Ok(CachedDist::from_remote(remote, filename, normalized_path)) } From 4c047f858f65150abdc80017347e388f4161d375 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Thu, 11 Jan 2024 10:11:07 -0500 Subject: [PATCH 5/6] Remove InMemoryWheel and dead code (#879) --- .../src/distribution_database.rs | 105 +++--------------- crates/puffin-distribution/src/download.rs | 24 ---- crates/puffin-distribution/src/lib.rs | 2 +- crates/puffin-distribution/src/unzip.rs | 9 +- 4 files changed, 16 insertions(+), 124 deletions(-) diff --git a/crates/puffin-distribution/src/distribution_database.rs b/crates/puffin-distribution/src/distribution_database.rs index 3baa5a9cc..e07d7dfea 100644 --- a/crates/puffin-distribution/src/distribution_database.rs +++ b/crates/puffin-distribution/src/distribution_database.rs @@ -4,7 +4,6 @@ use std::path::Path; use std::str::FromStr; use std::sync::Arc; -use bytesize::ByteSize; use fs_err::tokio as fs; use puffin_extract::unzip_no_seek; use thiserror::Error; @@ -25,9 +24,7 @@ use pypi_types::Metadata21; use crate::download::{BuiltWheel, UnzippedWheel}; use crate::locks::Locks; use crate::reporter::Facade; -use crate::{ - DiskWheel, InMemoryWheel, LocalWheel, Reporter, SourceDistCachedBuilder, SourceDistError, -}; +use crate::{DiskWheel, LocalWheel, Reporter, SourceDistCachedBuilder, SourceDistError}; #[derive(Debug, Error)] pub enum DistributionDatabaseError { @@ -124,11 +121,11 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context> wheel_filename.stem(), ); - // Start the download - let reader = self.client.stream_external(&url).await?; - - // In all wheels we've seen so far, unzipping while downloading is the - // faster option. + // Download and unzip on the same tokio task + // + // In all wheels we've seen so far, unzipping while downloading is + // faster than downloading into a file and then unzipping on multiple + // threads. // // Writing to a file first may be faster if the wheel takes longer to // unzip than it takes to download. This may happen if the wheel is a @@ -139,89 +136,15 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context> // for downloading and unzipping (with a buffer in between) and switch // to rayon if this buffer grows large by the time the file is fully // downloaded. - let unzip_while_downloading = true; - if unzip_while_downloading { - // Download and unzip to a temporary dir - let temp_dir = tempfile::tempdir_in(self.cache.root())?; - let temp_target = temp_dir.path().join(&wheel.file.filename); - unzip_no_seek(reader.compat(), &temp_target).await?; + let reader = self.client.stream_external(&url).await?; + let target = cache_entry.into_path_buf(); + unzip_no_seek(reader.compat(), &target).await?; - // Move the dir to the right place - fs::create_dir_all(&cache_entry.dir()).await?; - let target = cache_entry.into_path_buf(); - tokio::fs::rename(temp_target, &target).await?; - - return Ok(LocalWheel::Unzipped(UnzippedWheel { - dist: dist.clone(), - target, - filename: wheel_filename, - })); - } - - // If the file is greater than 5MB, write it to disk; otherwise, keep it in memory. - // - // TODO this is currently dead code. Consider deleting if there's no use for it. - let byte_size = wheel.file.size.map(ByteSize::b); - let local_wheel = if let Some(byte_size) = - byte_size.filter(|byte_size| *byte_size < ByteSize::mb(5)) - { - debug!("Fetching in-memory wheel from registry: {dist} ({byte_size})",); - - // Read into a buffer. - let mut buffer = Vec::with_capacity( - wheel - .file - .size - .unwrap_or(0) - .try_into() - .expect("5MB shouldn't be bigger usize::MAX"), - ); - let mut reader = tokio::io::BufReader::new(reader.compat()); - tokio::io::copy(&mut reader, &mut buffer).await?; - - LocalWheel::InMemory(InMemoryWheel { - dist: dist.clone(), - target: cache_entry.into_path_buf(), - buffer, - filename: wheel_filename, - }) - } else { - let size = - byte_size.map_or("unknown size".to_string(), |size| size.to_string()); - - debug!("Fetching disk-based wheel from registry: {dist} ({size})"); - - let filename = wheel_filename.to_string(); - - // Download the wheel to a temporary file. - let temp_dir = tempfile::tempdir_in(self.cache.root())?; - let temp_file = temp_dir.path().join(&filename); - let mut writer = - tokio::io::BufWriter::new(tokio::fs::File::create(&temp_file).await?); - tokio::io::copy(&mut reader.compat(), &mut writer).await?; - - // Move the temporary file to the cache. - let cache_entry = self.cache.entry( - CacheBucket::Wheels, - WheelCache::Index(&wheel.index) - .remote_wheel_dir(wheel_filename.name.as_ref()), - filename, // TODO should this be filename.stem() to match the other branch? - ); - fs::create_dir_all(&cache_entry.dir()).await?; - tokio::fs::rename(temp_file, &cache_entry.path()).await?; - - LocalWheel::Disk(DiskWheel { - dist: dist.clone(), - target: cache_entry - .with_file(wheel_filename.stem()) - .path() - .to_path_buf(), - path: cache_entry.into_path_buf(), - filename: wheel_filename, - }) - }; - - Ok(local_wheel) + Ok(LocalWheel::Unzipped(UnzippedWheel { + dist: dist.clone(), + target, + filename: wheel_filename, + })) } Dist::Built(BuiltDist::DirectUrl(wheel)) => { diff --git a/crates/puffin-distribution/src/download.rs b/crates/puffin-distribution/src/download.rs index b3ca4a98d..30f7f12c7 100644 --- a/crates/puffin-distribution/src/download.rs +++ b/crates/puffin-distribution/src/download.rs @@ -21,19 +21,6 @@ pub struct UnzippedWheel { pub(crate) target: PathBuf, } -/// A downloaded wheel that's stored in-memory. -#[derive(Debug, Clone)] -pub struct InMemoryWheel { - /// The remote distribution from which this wheel was downloaded. - pub(crate) dist: Dist, - /// The parsed filename. - pub(crate) filename: WheelFilename, - /// The contents of the wheel. - pub(crate) buffer: Vec, - /// The expected path to the downloaded wheel's entry in the cache. - pub(crate) target: PathBuf, -} - /// A downloaded wheel that's stored on-disk. #[derive(Debug, Clone)] pub struct DiskWheel { @@ -64,7 +51,6 @@ pub struct BuiltWheel { #[derive(Debug, Clone)] pub enum LocalWheel { Unzipped(UnzippedWheel), - InMemory(InMemoryWheel), Disk(DiskWheel), Built(BuiltWheel), } @@ -74,7 +60,6 @@ impl LocalWheel { pub fn target(&self) -> &Path { match self { LocalWheel::Unzipped(wheel) => &wheel.target, - LocalWheel::InMemory(wheel) => &wheel.target, LocalWheel::Disk(wheel) => &wheel.target, LocalWheel::Built(wheel) => &wheel.target, } @@ -84,7 +69,6 @@ impl LocalWheel { pub fn remote(&self) -> &Dist { match self { LocalWheel::Unzipped(wheel) => wheel.remote(), - LocalWheel::InMemory(wheel) => wheel.remote(), LocalWheel::Disk(wheel) => wheel.remote(), LocalWheel::Built(wheel) => wheel.remote(), } @@ -94,7 +78,6 @@ impl LocalWheel { pub fn filename(&self) -> &WheelFilename { match self { LocalWheel::Unzipped(wheel) => &wheel.filename, - LocalWheel::InMemory(wheel) => &wheel.filename, LocalWheel::Disk(wheel) => &wheel.filename, LocalWheel::Built(wheel) => &wheel.filename, } @@ -115,13 +98,6 @@ impl DiskWheel { } } -impl InMemoryWheel { - /// Return the [`Dist`] from which this wheel was downloaded. - pub fn remote(&self) -> &Dist { - &self.dist - } -} - impl BuiltWheel { /// Return the [`Dist`] from which this source distribution that this wheel was built from was /// downloaded. diff --git a/crates/puffin-distribution/src/lib.rs b/crates/puffin-distribution/src/lib.rs index 25507c235..134b9d19b 100644 --- a/crates/puffin-distribution/src/lib.rs +++ b/crates/puffin-distribution/src/lib.rs @@ -1,5 +1,5 @@ pub use distribution_database::{DistributionDatabase, DistributionDatabaseError}; -pub use download::{BuiltWheel, DiskWheel, InMemoryWheel, LocalWheel}; +pub use download::{BuiltWheel, DiskWheel, LocalWheel}; pub use index::{BuiltWheelIndex, RegistryWheelIndex}; pub use reporter::Reporter; pub use source::{SourceDistCachedBuilder, SourceDistError}; diff --git a/crates/puffin-distribution/src/unzip.rs b/crates/puffin-distribution/src/unzip.rs index 6708cc576..37a8f6843 100644 --- a/crates/puffin-distribution/src/unzip.rs +++ b/crates/puffin-distribution/src/unzip.rs @@ -3,19 +3,13 @@ use std::path::Path; use puffin_extract::{unzip_archive, Error}; use crate::download::BuiltWheel; -use crate::{DiskWheel, InMemoryWheel, LocalWheel}; +use crate::{DiskWheel, LocalWheel}; pub trait Unzip { /// Unzip a wheel into the target directory. fn unzip(&self, target: &Path) -> Result<(), Error>; } -impl Unzip for InMemoryWheel { - fn unzip(&self, target: &Path) -> Result<(), Error> { - unzip_archive(std::io::Cursor::new(&self.buffer), target) - } -} - impl Unzip for DiskWheel { fn unzip(&self, target: &Path) -> Result<(), Error> { unzip_archive(fs_err::File::open(&self.path)?, target) @@ -32,7 +26,6 @@ impl Unzip for LocalWheel { fn unzip(&self, target: &Path) -> Result<(), Error> { match self { LocalWheel::Unzipped(_) => Ok(()), - LocalWheel::InMemory(wheel) => wheel.unzip(target), LocalWheel::Disk(wheel) => wheel.unzip(target), LocalWheel::Built(wheel) => wheel.unzip(target), } From 90edfa8fe7a6f678e7b536bf9271ab65467b3eca Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 11 Jan 2024 12:28:53 -0600 Subject: [PATCH 6/6] Use `mold` for linking in CI tests (#887) Derived from https://github.com/astral-sh/puffin/pull/875 This gets us a significant speedup. I would not read the commits individually. I can squash them but they were used for testing various scenarios. ### Test compile times Ranges are the lowest and highest I've seen. Huge variability in GitHub Actions runners. **Before:** 7m 21s - 8m 22s (cold cache) 110s - 120s (warm cache) **After:** 6m 15s - 7m 05s (cold cache) 57s - 70s (warm cache) **Improvement:** 4% - 25% (cold cache) 36% - 52% (warm cache) --- .github/workflows/ci.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8ce128dd6..1a11b32ae 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,7 +2,7 @@ name: CI on: push: - branches: [ main ] + branches: [main] pull_request: workflow_dispatch: @@ -45,7 +45,7 @@ jobs: cargo-test: strategy: matrix: - os: [ ubuntu-latest ] + os: [ubuntu-latest] runs-on: ${{ matrix.os }} name: "cargo test | ${{ matrix.os }}" steps: @@ -53,7 +53,7 @@ jobs: - name: "Install Python" uses: actions/setup-python@v4 with: - python-version: | + python-version: | 3.7 3.8 3.9 @@ -62,6 +62,7 @@ jobs: 3.12 - name: "Install Rust toolchain" run: rustup show + - uses: rui314/setup-mold@v1 - name: "Install cargo insta" uses: taiki-e/install-action@v2 with: