diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1c32a9e84..f85f20b0a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,7 +38,7 @@ jobs: rustup component add clippy - uses: Swatinem/rust-cache@v2 - name: "Clippy" - run: cargo clippy --workspace --all-targets --all-features -- -D warnings + run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings cargo-test: strategy: diff --git a/Cargo.lock b/Cargo.lock index 0cfd0118b..77a309714 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -557,22 +557,21 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" -[[package]] -name = "colored" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" -dependencies = [ - "lazy_static", - "windows-sys 0.48.0", -] - [[package]] name = "configparser" version = "3.0.4" @@ -926,6 +925,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", + "libz-ng-sys", "miniz_oxide", ] @@ -1659,6 +1659,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-ng-sys" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81157dde2fd4ad2b45ea3a4bb47b8193b52a6346b678840d91d80d3c2cd166c5" +dependencies = [ + "cmake", + "libc", +] + [[package]] name = "libz-sys" version = "1.1.14" @@ -2250,7 +2260,7 @@ dependencies = [ [[package]] name = "pubgrub" version = "0.2.1" -source = "git+https://github.com/zanieb/pubgrub?rev=78b8add6942766e5fb070bbda1de570e93d6399f#78b8add6942766e5fb070bbda1de570e93d6399f" +source = "git+https://github.com/zanieb/pubgrub?rev=866c0f2a87fee1e8abe804d40a2ee934de0973d7#866c0f2a87fee1e8abe804d40a2ee934de0973d7" dependencies = [ "indexmap 2.1.0", "log", @@ -2319,7 +2329,6 @@ dependencies = [ "bitflags 2.4.1", "chrono", "clap", - "colored", "distribution-filename", "distribution-types", "fs-err", @@ -2333,6 +2342,7 @@ dependencies = [ "itertools 0.12.0", "miette", "mimalloc", + "owo-colors", "pep440_rs 0.3.12", "pep508_rs", "platform-host", @@ -2363,6 +2373,7 @@ dependencies = [ "tokio", "toml", "tracing", + "tracing-durations-export", "tracing-subscriber", "tracing-tree", "url", @@ -2417,7 +2428,6 @@ dependencies = [ "anyhow", "chrono", "clap", - "colored", "distribution-filename", "distribution-types", "fs-err", @@ -2427,6 +2437,7 @@ dependencies = [ "install-wheel-rs", "itertools 0.12.0", "mimalloc", + "owo-colors", "pep440_rs 0.3.12", "pep508_rs", "petgraph", @@ -2448,6 +2459,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "tracing", + "tracing-durations-export", "tracing-indicatif", "tracing-subscriber", "url", @@ -2631,12 +2643,12 @@ dependencies = [ name = "puffin-resolver" version = "0.0.1" dependencies = [ + "anstream", "anyhow", "bitflags 2.4.1", "cache-key", "chrono", "clap", - "colored", "derivative", "distribution-filename", "distribution-types", @@ -2648,6 +2660,7 @@ dependencies = [ "install-wheel-rs", "itertools 0.12.0", "once_cell", + "owo-colors", "pep440_rs 0.3.12", "pep508_rs", "petgraph", @@ -2697,8 +2710,8 @@ name = "puffin-warnings" version = "0.0.1" dependencies = [ "anstream", - "colored", "once_cell", + "owo-colors", "rustc-hash", ] @@ -3451,6 +3464,12 @@ dependencies = [ "is-terminal", ] +[[package]] +name = "svg" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d703a3635418d4e4d0e410009ddbfb65047ef9468b1d29afd3b057a5bc4c217" + [[package]] name = "syn" version = "1.0.109" @@ -3860,6 +3879,24 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-durations-export" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6bb8898f56f636911130c78cc528338a2bb0426bdfb5a8fb523f98fc8da46d" +dependencies = [ + "anyhow", + "fs-err", + "itertools 0.12.0", + "once_cell", + "rustc-hash", + "serde", + "serde_json", + "svg", + "tracing", + "tracing-subscriber", +] + [[package]] name = "tracing-indicatif" version = "0.3.6" diff --git a/Cargo.toml b/Cargo.toml index cf840ad22..f74838c44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,14 +24,16 @@ camino = { version = "1.1.6", features = ["serde1"] } cargo-util = { version = "0.2.8" } chrono = { version = "0.4.31" } clap = { version = "4.4.13" } -colored = { version = "2.1.0" } configparser = { version = "3.0.4" } csv = { version = "1.3.0" } data-encoding = { version = "2.5.0" } derivative = { version = "2.2.0" } directories = { version = "5.0.1" } dirs = { version = "5.0.1" } -flate2 = { version = "1.0.28" } +# This tells flate2 (and all libraries that depend on it, including async_compression +# and async_zip) to use zlib-ng, which about 2x faster than the default flate2 backend +# at decompression. See https://github.com/rust-lang/flate2-rs#backends +flate2 = { version = "1.0.28", features = ["zlib-ng"], default-features = false } fs-err = { version = "2.11.0" } fs2 = { version = "0.4.3" } futures = { version = "0.3.30" } @@ -49,10 +51,11 @@ mailparse = { version = "0.14.0" } # For additional textwrap options: https://github.com/zkat/miette/pull/321, https://github.com/zkat/miette/pull/328 miette = { git = "https://github.com/zkat/miette.git", rev = "b0744462adbbfbb6d845f382db36be883c7f3c45" } once_cell = { version = "1.19.0" } +owo-colors = { version = "3.5.0" } petgraph = { version = "0.6.4" } platform-info = { version = "2.0.2" } plist = { version = "1.6.0" } -pubgrub = { git = "https://github.com/zanieb/pubgrub", rev = "78b8add6942766e5fb070bbda1de570e93d6399f" } +pubgrub = { git = "https://github.com/zanieb/pubgrub", rev = "866c0f2a87fee1e8abe804d40a2ee934de0973d7" } pyo3 = { version = "0.20.2" } pyo3-log = { version = "0.9.0"} pyproject-toml = { version = "0.8.1" } @@ -81,6 +84,7 @@ tokio-util = { version = "0.7.10", features = ["compat"] } toml = { version = "0.8.8" } toml_edit = { version = "0.21.0" } tracing = { version = "0.1.40" } +tracing-durations-export = { version = "0.1.0", features = ["plot"] } tracing-indicatif = { version = "0.3.6" } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } tracing-tree = { version = "0.3.0" } diff --git a/crates/distribution-types/src/cached.rs b/crates/distribution-types/src/cached.rs index f0e4a0ed6..a37d510d6 100644 --- a/crates/distribution-types/src/cached.rs +++ b/crates/distribution-types/src/cached.rs @@ -141,23 +141,16 @@ pub struct CachedWheel { impl CachedWheel { /// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any`). - pub fn from_path(path: &Path) -> Result> { - let Some(file_name) = path.file_name() else { - return Ok(None); - }; - let Some(file_name) = file_name.to_str() else { - return Ok(None); - }; - let Ok(filename) = WheelFilename::from_stem(file_name) else { - return Ok(None); - }; + pub fn from_path(path: &Path) -> Option { + let filename = path.file_name()?.to_str()?; + let filename = WheelFilename::from_stem(filename).ok()?; if path.is_file() { - return Ok(None); + return None; } let path = path.to_path_buf(); - Ok(Some(Self { filename, path })) + Some(Self { filename, path }) } /// Convert a [`CachedWheel`] into a [`CachedRegistryDist`]. diff --git a/crates/distribution-types/src/file.rs b/crates/distribution-types/src/file.rs index 1e8a1b3d3..69ca46b9d 100644 --- a/crates/distribution-types/src/file.rs +++ b/crates/distribution-types/src/file.rs @@ -1,7 +1,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use pep440_rs::VersionSpecifiers; +use pep440_rs::{VersionSpecifiers, VersionSpecifiersParseError}; use pypi_types::{DistInfoMetadata, Hashes, Yanked}; /// Internal analog to [`pypi_types::File`]. @@ -11,23 +11,26 @@ pub struct File { pub filename: String, pub hashes: Hashes, pub requires_python: Option, - pub size: Option, + pub size: Option, pub upload_time: Option>, pub url: String, pub yanked: Option, } -impl From for File { - fn from(file: pypi_types::File) -> Self { - Self { +impl TryFrom for File { + type Error = VersionSpecifiersParseError; + + /// `TryFrom` instead of `From` to filter out files with invalid requires python version specifiers + fn try_from(file: pypi_types::File) -> Result { + Ok(Self { dist_info_metadata: file.dist_info_metadata, filename: file.filename, hashes: file.hashes, - requires_python: file.requires_python, + requires_python: file.requires_python.transpose()?, size: file.size, upload_time: file.upload_time, url: file.url, yanked: file.yanked, - } + }) } } diff --git a/crates/distribution-types/src/lib.rs b/crates/distribution-types/src/lib.rs index 64e7f167c..1e3a06082 100644 --- a/crates/distribution-types/src/lib.rs +++ b/crates/distribution-types/src/lib.rs @@ -506,7 +506,7 @@ impl RemoteSource for File { Ok(&self.filename) } - fn size(&self) -> Option { + fn size(&self) -> Option { self.size } } @@ -518,7 +518,7 @@ impl RemoteSource for Url { .ok_or_else(|| Error::UrlFilename(self.clone())) } - fn size(&self) -> Option { + fn size(&self) -> Option { None } } @@ -528,7 +528,7 @@ impl RemoteSource for RegistryBuiltDist { self.file.filename() } - fn size(&self) -> Option { + fn size(&self) -> Option { self.file.size() } } @@ -538,7 +538,7 @@ impl RemoteSource for RegistrySourceDist { self.file.filename() } - fn size(&self) -> Option { + fn size(&self) -> Option { self.file.size() } } @@ -548,7 +548,7 @@ impl RemoteSource for DirectUrlBuiltDist { self.url.filename() } - fn size(&self) -> Option { + fn size(&self) -> Option { self.url.size() } } @@ -558,7 +558,7 @@ impl RemoteSource for DirectUrlSourceDist { self.url.filename() } - fn size(&self) -> Option { + fn size(&self) -> Option { self.url.size() } } @@ -572,7 +572,7 @@ impl RemoteSource for GitSourceDist { }) } - fn size(&self) -> Option { + fn size(&self) -> Option { self.url.size() } } @@ -582,7 +582,7 @@ impl RemoteSource for PathBuiltDist { self.url.filename() } - fn size(&self) -> Option { + fn size(&self) -> Option { self.url.size() } } @@ -592,7 +592,7 @@ impl RemoteSource for PathSourceDist { self.url.filename() } - fn size(&self) -> Option { + fn size(&self) -> Option { self.url.size() } } @@ -607,7 +607,7 @@ impl RemoteSource for SourceDist { } } - fn size(&self) -> Option { + fn size(&self) -> Option { match self { Self::Registry(dist) => dist.size(), Self::DirectUrl(dist) => dist.size(), @@ -626,7 +626,7 @@ impl RemoteSource for BuiltDist { } } - fn size(&self) -> Option { + fn size(&self) -> Option { match self { Self::Registry(dist) => dist.size(), Self::DirectUrl(dist) => dist.size(), @@ -643,7 +643,7 @@ impl RemoteSource for Dist { } } - fn size(&self) -> Option { + fn size(&self) -> Option { match self { Self::Built(dist) => dist.size(), Self::Source(dist) => dist.size(), diff --git a/crates/distribution-types/src/traits.rs b/crates/distribution-types/src/traits.rs index 705b15e8c..e4a1ed2f6 100644 --- a/crates/distribution-types/src/traits.rs +++ b/crates/distribution-types/src/traits.rs @@ -53,7 +53,7 @@ pub trait RemoteSource { fn filename(&self) -> Result<&str, Error>; /// Return the size of the distribution, if known. - fn size(&self) -> Option; + fn size(&self) -> Option; } pub trait Identifier { diff --git a/crates/gourgeist/Cargo.toml b/crates/gourgeist/Cargo.toml index 6cdcc05f9..6f06028b0 100644 --- a/crates/gourgeist/Cargo.toml +++ b/crates/gourgeist/Cargo.toml @@ -23,7 +23,7 @@ puffin-interpreter = { path = "../puffin-interpreter" } anstream = { workspace = true } camino = { workspace = true } -clap = { workspace = true } +clap = { workspace = true, features = ["derive"] } fs-err = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/install-wheel-rs/src/linker.rs b/crates/install-wheel-rs/src/linker.rs index 162a6da72..ec016684e 100644 --- a/crates/install-wheel-rs/src/linker.rs +++ b/crates/install-wheel-rs/src/linker.rs @@ -6,7 +6,7 @@ use std::path::Path; use configparser::ini::Ini; use fs_err as fs; use fs_err::File; -use tracing::{debug, info_span}; +use tracing::{debug, instrument}; use pypi_types::DirectUrl; @@ -24,6 +24,7 @@ use crate::{read_record_file, Error, Script}; /// /// /// Wheel 1.0: +#[instrument(skip_all, fields(wheel = %wheel.as_ref().display()))] pub fn install_wheel( location: &InstallLocation>, wheel: impl AsRef, @@ -52,8 +53,6 @@ pub fn install_wheel( let metadata = dist_info_metadata(&dist_info_prefix, &wheel)?; let (name, _version) = parse_metadata(&dist_info_prefix, &metadata)?; - let _my_span = info_span!("install_wheel", name); - // We're going step by step though // https://packaging.python.org/en/latest/specifications/binary-distribution-format/#installing-a-wheel-distribution-1-0-py32-none-any-whl // > 1.a Parse distribution-1.0.dist-info/WHEEL. @@ -137,8 +136,9 @@ fn find_dist_info(path: impl AsRef) -> Result { // Iterate over `path` to find the `.dist-info` directory. It should be at the top-level. let Some(dist_info) = fs::read_dir(path.as_ref())?.find_map(|entry| { let entry = entry.ok()?; - let path = entry.path(); - if path.is_dir() { + let file_type = entry.file_type().ok()?; + if file_type.is_dir() { + let path = entry.path(); if path.extension().map_or(false, |ext| ext == "dist-info") { Some(path) } else { @@ -231,6 +231,7 @@ impl Default for LinkMode { impl LinkMode { /// Extract a wheel by linking all of its files into site packages. + #[instrument(skip_all)] pub fn link_wheel_files( self, site_packages: impl AsRef, @@ -317,7 +318,9 @@ fn copy_wheel_files( // Walk over the directory. for entry in walkdir::WalkDir::new(&wheel) { let entry = entry?; - let relative = entry.path().strip_prefix(&wheel).unwrap(); + let path = entry.path(); + + let relative = path.strip_prefix(&wheel).unwrap(); let out_path = site_packages.as_ref().join(relative); if entry.file_type().is_dir() { @@ -326,7 +329,7 @@ fn copy_wheel_files( } // Copy the file. - fs::copy(entry.path(), &out_path)?; + fs::copy(path, &out_path)?; #[cfg(unix)] { @@ -370,7 +373,9 @@ fn hardlink_wheel_files( // Walk over the directory. for entry in walkdir::WalkDir::new(&wheel) { let entry = entry?; - let relative = entry.path().strip_prefix(&wheel).unwrap(); + let path = entry.path(); + + let relative = path.strip_prefix(&wheel).unwrap(); let out_path = site_packages.as_ref().join(relative); if entry.file_type().is_dir() { @@ -379,8 +384,8 @@ fn hardlink_wheel_files( } // The `RECORD` file is modified during installation, so we copy it instead of hard-linking. - if entry.path().ends_with("RECORD") { - fs::copy(entry.path(), &out_path)?; + if path.ends_with("RECORD") { + fs::copy(path, &out_path)?; count += 1; continue; } @@ -390,33 +395,33 @@ fn hardlink_wheel_files( Attempt::Initial => { // Once https://github.com/rust-lang/rust/issues/86442 is stable, use that. attempt = Attempt::Subsequent; - if let Err(err) = fs::hard_link(entry.path(), &out_path) { + if let Err(err) = fs::hard_link(path, &out_path) { // If the file already exists, remove it and try again. if err.kind() == std::io::ErrorKind::AlreadyExists { fs::remove_file(&out_path)?; - if fs::hard_link(entry.path(), &out_path).is_err() { - fs::copy(entry.path(), &out_path)?; + if fs::hard_link(path, &out_path).is_err() { + fs::copy(path, &out_path)?; attempt = Attempt::UseCopyFallback; } } else { - fs::copy(entry.path(), &out_path)?; + fs::copy(path, &out_path)?; attempt = Attempt::UseCopyFallback; } } } Attempt::Subsequent => { - if let Err(err) = fs::hard_link(entry.path(), &out_path) { + if let Err(err) = fs::hard_link(path, &out_path) { // If the file already exists, remove it and try again. if err.kind() == std::io::ErrorKind::AlreadyExists { fs::remove_file(&out_path)?; - fs::hard_link(entry.path(), &out_path)?; + fs::hard_link(path, &out_path)?; } else { return Err(err.into()); } } } Attempt::UseCopyFallback => { - fs::copy(entry.path(), &out_path)?; + fs::copy(path, &out_path)?; } } diff --git a/crates/install-wheel-rs/src/wheel.rs b/crates/install-wheel-rs/src/wheel.rs index d5e0c009c..651c90138 100644 --- a/crates/install-wheel-rs/src/wheel.rs +++ b/crates/install-wheel-rs/src/wheel.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::ffi::OsString; + use std::io::{BufRead, BufReader, BufWriter, Cursor, Read, Seek, Write}; use std::path::{Path, PathBuf}; use std::process::{Command, ExitStatus, Stdio}; @@ -87,9 +87,8 @@ fn parse_scripts( ) -> Result<(Vec