From e9b6b6fa36b7ba587b5207cdc96a1cbe34440add Mon Sep 17 00:00:00 2001 From: konsti Date: Mon, 15 Jan 2024 03:04:10 +0100 Subject: [PATCH] Implement `--find-links` as flat indexes (directories in pip-compile) (#912) Add directory `--find-links` support for local paths to pip-compile. It seems that pip joins all sources and then picks the best package. We explicitly give find links packages precedence if the same exists on an index and locally by prefilling the `VersionMap`, otherwise they are added as another index and the existing rules of precedence apply. Internally, the feature is called _flat index_, which is more meaningful than _find links_: We're not looking for links, we're picking up local directories, and (TBD) support another index format that's just a flat list of files instead of a nested index. `RegistryBuiltDist` and `RegistrySourceDist` now use `WheelFilename` and `SourceDistFilename` respectively. The `File` inside `RegistryBuiltDist` and `RegistrySourceDist` gained the ability to represent both a url and a path so that `--find-links` with a url and with a path works the same, both being locked as `@` instead of ` @ `. (This is more of a detail, this PR in general still work if we strip that and have directory find links represented as ` @ file:///path/to/file.ext`) `PrioritizedDistribution` and `FlatIndex` have been moved to locations where we can use them in the upstack PR. I added a `scripts/wheels` directory with stripped down wheels to use for testing. We're lacking tests for correct tag priority precedence with flat indexes, i only confirmed this manually since it is not covered in the pip-compile or pip-sync output. Closes #876 --- Cargo.lock | 4 + crates/distribution-filename/src/lib.rs | 46 +++- .../distribution-filename/src/source_dist.rs | 42 ++- crates/distribution-types/Cargo.toml | 3 +- crates/distribution-types/src/file.rs | 31 ++- crates/distribution-types/src/index_url.rs | 78 ++++-- crates/distribution-types/src/lib.rs | 94 ++++--- .../src/prioritized_distribution.rs | 207 ++++++++++++++ crates/distribution-types/src/resolution.rs | 8 +- crates/pep508-rs/src/verbatim_url.rs | 2 + crates/puffin-cli/src/commands/pip_compile.rs | 10 +- crates/puffin-cli/src/commands/pip_install.rs | 17 +- crates/puffin-cli/src/commands/pip_sync.rs | 12 +- crates/puffin-cli/src/commands/venv.rs | 10 +- crates/puffin-cli/src/main.rs | 64 ++++- crates/puffin-cli/tests/pip_compile.rs | 53 ++++ crates/puffin-client/Cargo.toml | 6 +- crates/puffin-client/src/error.rs | 5 +- crates/puffin-client/src/flat_index.rs | 104 +++++++ crates/puffin-client/src/lib.rs | 2 + crates/puffin-client/src/registry_client.rs | 255 ++++++++++++------ crates/puffin-dev/src/build.rs | 4 +- crates/puffin-dev/src/install_many.rs | 12 +- crates/puffin-dev/src/resolve_cli.rs | 22 +- crates/puffin-dev/src/resolve_many.rs | 6 +- crates/puffin-dispatch/src/lib.rs | 12 +- .../src/distribution_database.rs | 24 +- .../src/index/registry_wheel_index.rs | 21 +- crates/puffin-distribution/src/source/mod.rs | 38 ++- crates/puffin-installer/src/downloader.rs | 2 +- crates/puffin-installer/src/plan.rs | 6 +- .../puffin-resolver/src/candidate_selector.rs | 17 +- crates/puffin-resolver/src/finder.rs | 22 +- crates/puffin-resolver/src/resolution.rs | 8 +- crates/puffin-resolver/src/resolver/mod.rs | 62 +++-- .../puffin-resolver/src/resolver/provider.rs | 19 +- crates/puffin-resolver/src/version_map.rs | 222 ++------------- crates/puffin-resolver/tests/resolver.rs | 2 +- scripts/wheels/maturin-1.4.0-py3-none-any.whl | Bin 0 -> 5943 bytes .../maturin-2.0.0-py3-none-linux_x86_64.whl | Bin 0 -> 5915 bytes scripts/wheels/tqdm-1000.0.0-py3-none-any.whl | Bin 0 -> 1017 bytes ...ylinux2010_x86_64.musllinux_1_1_x86_64.whl | Bin 0 -> 5579 bytes 42 files changed, 1069 insertions(+), 483 deletions(-) create mode 100644 crates/distribution-types/src/prioritized_distribution.rs create mode 100644 crates/puffin-client/src/flat_index.rs create mode 100644 scripts/wheels/maturin-1.4.0-py3-none-any.whl create mode 100644 scripts/wheels/maturin-2.0.0-py3-none-linux_x86_64.whl create mode 100644 scripts/wheels/tqdm-1000.0.0-py3-none-any.whl create mode 100644 scripts/wheels/tqdm-4.66.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl diff --git a/Cargo.lock b/Cargo.lock index becd72598..da1d0575b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -854,6 +854,7 @@ dependencies = [ "once_cell", "pep440_rs 0.3.12", "pep508_rs", + "platform-tags", "puffin-git", "puffin-normalize", "pypi-types", @@ -2403,6 +2404,7 @@ dependencies = [ "async_http_range_reader", "async_zip", "cache-key", + "chrono", "distribution-filename", "distribution-types", "fs-err", @@ -2414,6 +2416,7 @@ dependencies = [ "install-wheel-rs", "pep440_rs 0.3.12", "pep508_rs", + "platform-tags", "puffin-cache", "puffin-fs", "puffin-normalize", @@ -2422,6 +2425,7 @@ dependencies = [ "reqwest-middleware", "reqwest-retry", "rmp-serde", + "rustc-hash", "serde", "serde_json", "sha2", diff --git a/crates/distribution-filename/src/lib.rs b/crates/distribution-filename/src/lib.rs index 7e4072f14..6a273394c 100644 --- a/crates/distribution-filename/src/lib.rs +++ b/crates/distribution-filename/src/lib.rs @@ -1,3 +1,6 @@ +use pep440_rs::Version; +use puffin_normalize::PackageName; +use std::fmt::{Display, Formatter}; use std::str::FromStr; pub use source_dist::{SourceDistExtension, SourceDistFilename, SourceDistFilenameError}; @@ -13,10 +16,8 @@ pub enum DistFilename { } impl DistFilename { - pub fn try_from_filename( - filename: &str, - package_name: &puffin_normalize::PackageName, - ) -> Option { + /// Parse a filename as wheel or source dist name. + pub fn try_from_filename(filename: &str, package_name: &PackageName) -> Option { if let Ok(filename) = WheelFilename::from_str(filename) { Some(Self::WheelFilename(filename)) } else if let Ok(filename) = SourceDistFilename::parse(filename, package_name) { @@ -25,4 +26,41 @@ impl DistFilename { None } } + + /// Like [`DistFilename::try_from_normalized_filename`], but without knowing the package name. + /// + /// Source dist filenames can be ambiguous, e.g. `a-1-1.tar.gz`. Without knowing the package name, we assume that + /// source dist filename version doesn't contain minus (the version is normalized). + pub fn try_from_normalized_filename(filename: &str) -> Option { + if let Ok(filename) = WheelFilename::from_str(filename) { + Some(Self::WheelFilename(filename)) + } else if let Ok(filename) = SourceDistFilename::parsed_normalized_filename(filename) { + Some(Self::SourceDistFilename(filename)) + } else { + None + } + } + + pub fn name(&self) -> &PackageName { + match self { + DistFilename::SourceDistFilename(filename) => &filename.name, + DistFilename::WheelFilename(filename) => &filename.name, + } + } + + pub fn version(&self) -> &Version { + match self { + DistFilename::SourceDistFilename(filename) => &filename.version, + DistFilename::WheelFilename(filename) => &filename.version, + } + } +} + +impl Display for DistFilename { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + DistFilename::SourceDistFilename(filename) => Display::fmt(filename, f), + DistFilename::WheelFilename(filename) => Display::fmt(filename, f), + } + } } diff --git a/crates/distribution-filename/src/source_dist.rs b/crates/distribution-filename/src/source_dist.rs index fa0cdc646..cf50aa78f 100644 --- a/crates/distribution-filename/src/source_dist.rs +++ b/crates/distribution-filename/src/source_dist.rs @@ -105,6 +105,43 @@ impl SourceDistFilename { extension, }) } + + /// Like [`SourceDistFilename::parse`], but without knowing the package name. + /// + /// Source dist filenames can be ambiguous, e.g. `a-1-1.tar.gz`. Without knowing the package name, we assume that + /// source dist filename version doesn't contain minus (the version is normalized). + pub fn parsed_normalized_filename(filename: &str) -> Result { + let Some((stem, extension)) = SourceDistExtension::from_filename(filename) else { + return Err(SourceDistFilenameError { + filename: filename.to_string(), + kind: SourceDistFilenameErrorKind::Extension, + }); + }; + + let Some((package_name, version)) = stem.rsplit_once('-') else { + return Err(SourceDistFilenameError { + filename: filename.to_string(), + kind: SourceDistFilenameErrorKind::Minus, + }); + }; + let package_name = + PackageName::from_str(package_name).map_err(|err| SourceDistFilenameError { + filename: filename.to_string(), + kind: SourceDistFilenameErrorKind::PackageName(err), + })?; + + // We checked the length above + let version = Version::from_str(version).map_err(|err| SourceDistFilenameError { + filename: filename.to_string(), + kind: SourceDistFilenameErrorKind::Version(err), + })?; + + Ok(Self { + name: package_name, + version, + extension, + }) + } } impl Display for SourceDistFilename { @@ -139,13 +176,16 @@ enum SourceDistFilenameErrorKind { Version(#[from] VersionParseError), #[error(transparent)] PackageName(#[from] InvalidNameError), + #[error("Missing name-version separator")] + Minus, } #[cfg(test)] mod tests { - use puffin_normalize::PackageName; use std::str::FromStr; + use puffin_normalize::PackageName; + use crate::SourceDistFilename; /// Only test already normalized names since the parsing is lossy diff --git a/crates/distribution-types/Cargo.toml b/crates/distribution-types/Cargo.toml index 023fc6406..4b87d2665 100644 --- a/crates/distribution-types/Cargo.toml +++ b/crates/distribution-types/Cargo.toml @@ -14,9 +14,10 @@ workspace = true [dependencies] cache-key = { path = "../cache-key" } -distribution-filename = { path = "../distribution-filename" } +distribution-filename = { path = "../distribution-filename", features = ["serde"] } pep440_rs = { path = "../pep440-rs" } pep508_rs = { path = "../pep508-rs" } +platform-tags = { path = "../platform-tags" } puffin-git = { path = "../puffin-git" } puffin-normalize = { path = "../puffin-normalize" } pypi-types = { path = "../pypi-types" } diff --git a/crates/distribution-types/src/file.rs b/crates/distribution-types/src/file.rs index b616ce059..cfb38be0c 100644 --- a/crates/distribution-types/src/file.rs +++ b/crates/distribution-types/src/file.rs @@ -1,9 +1,13 @@ +use std::fmt::{Display, Formatter}; +use std::path::PathBuf; + use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use thiserror::Error; use url::Url; use pep440_rs::{VersionSpecifiers, VersionSpecifiersParseError}; +use pep508_rs::VerbatimUrl; use pypi_types::{BaseUrl, DistInfoMetadata, Hashes, Yanked}; /// Error converting [`pypi_types::File`] to [`distribution_type::File`]. @@ -24,7 +28,7 @@ pub struct File { pub requires_python: Option, pub size: Option, pub upload_time: Option>, - pub url: Url, + pub url: FileLocation, pub yanked: Option, } @@ -38,10 +42,29 @@ impl File { requires_python: file.requires_python.transpose()?, size: file.size, upload_time: file.upload_time, - url: base - .join_relative(&file.url) - .map_err(|err| FileConversionError::Url(file.url.clone(), err))?, + url: FileLocation::Url( + base.join_relative(&file.url) + .map_err(|err| FileConversionError::Url(file.url.clone(), err))?, + ), yanked: file.yanked, }) } } + +/// While a registry file is generally a remote URL, it can also be a file if it comes from a directory flat indexes. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum FileLocation { + /// URL relative to base + Url(Url), + /// Absolute path to file + Path(PathBuf, VerbatimUrl), +} + +impl Display for FileLocation { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + FileLocation::Url(url) => Display::fmt(url, f), + FileLocation::Path(path, _url) => Display::fmt(&path.display(), f), + } + } +} diff --git a/crates/distribution-types/src/index_url.rs b/crates/distribution-types/src/index_url.rs index 46b97551f..ff1779c5d 100644 --- a/crates/distribution-types/src/index_url.rs +++ b/crates/distribution-types/src/index_url.rs @@ -1,5 +1,5 @@ -use std::iter::Chain; use std::ops::Deref; +use std::path::PathBuf; use std::str::FromStr; use once_cell::sync::Lazy; @@ -53,6 +53,44 @@ impl Deref for IndexUrl { } } +/// A directory with distributions or a URL to an HTML file with a flat listing of distributions. +/// +/// Also known as `--find-links`. +#[derive(Debug, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)] +pub enum FlatIndexLocation { + Path(PathBuf), + Url(Url), +} + +impl FromStr for FlatIndexLocation { + type Err = FlatIndexError; + + fn from_str(location: &str) -> Result { + if location.contains("://") { + let url = + Url::parse(location).map_err(|err| FlatIndexError::Url(location.into(), err))?; + if url.scheme() == "file" { + match url.to_file_path() { + Ok(path_buf) => Ok(Self::Path(path_buf)), + Err(()) => Err(FlatIndexError::FilePath(url)), + } + } else { + Ok(Self::Url(url)) + } + } else { + Ok(Self::Path(PathBuf::from(location))) + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum FlatIndexError { + #[error("Invalid file location URL: {0}")] + Url(String, #[source] url::ParseError), + #[error("Invalid `file://` path in URL: {0}")] + FilePath(Url), +} + /// The index URLs to use for fetching packages. /// /// "pip treats all package sources equally" (), @@ -60,34 +98,45 @@ impl Deref for IndexUrl { /// /// If the fields are none and empty, ignore the package index, instead rely on local archives and /// caches. +/// +/// From a pip perspective, this type merges `--index-url`, `--extra-index-url`, and `--find-links`. #[derive(Debug, Clone)] -pub struct IndexUrls { - pub index: Option, - pub extra_index: Vec, +pub struct IndexLocations { + index: Option, + extra_index: Vec, + flat_index: Vec, } -impl Default for IndexUrls { +impl Default for IndexLocations { /// Just pypi fn default() -> Self { Self { index: Some(IndexUrl::Pypi), extra_index: Vec::new(), + flat_index: Vec::new(), } } } -impl IndexUrls { +impl IndexLocations { /// Determine the index URLs to use for fetching packages. - pub fn from_args(index: IndexUrl, extra_index: Vec, no_index: bool) -> Self { + pub fn from_args( + index: IndexUrl, + extra_index: Vec, + flat_index: Vec, + no_index: bool, + ) -> Self { if no_index { Self { index: None, extra_index: Vec::new(), + flat_index: Vec::new(), } } else { Self { index: Some(index), extra_index, + flat_index, } } } @@ -97,19 +146,12 @@ impl IndexUrls { } } -impl<'a> IntoIterator for &'a IndexUrls { - type Item = &'a IndexUrl; - type IntoIter = Chain, std::slice::Iter<'a, IndexUrl>>; - - fn into_iter(self) -> Self::IntoIter { +impl<'a> IndexLocations { + pub fn indexes(&'a self) -> impl Iterator + 'a { self.index.iter().chain(self.extra_index.iter()) } -} -impl<'a> IndexUrls { - pub fn iter( - &'a self, - ) -> Chain, std::slice::Iter<'a, IndexUrl>> { - self.into_iter() + pub fn flat_indexes(&'a self) -> impl Iterator + 'a { + self.flat_index.iter() } } diff --git a/crates/distribution-types/src/lib.rs b/crates/distribution-types/src/lib.rs index da417daeb..e3b0b64a8 100644 --- a/crates/distribution-types/src/lib.rs +++ b/crates/distribution-types/src/lib.rs @@ -25,8 +25,6 @@ //! * [`CachedRegistryDist`] //! * [`CachedDirectUrlDist`] //! -//! TODO(konstin): Track all kinds from [`Dist`]. -//! //! ## `InstalledDist` //! An [`InstalledDist`] is built distribution (wheel) that is installed in a virtual environment, //! with the two possible origins we currently track: @@ -34,8 +32,6 @@ //! * [`InstalledDirectUrlDist`] //! //! Since we read this information from [`direct_url.json`](https://packaging.python.org/en/latest/specifications/direct-url-data-structure/), it doesn't match the information [`Dist`] exactly. -//! -//! TODO(konstin): Track all kinds from [`Dist`]. use std::borrow::Cow; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -43,7 +39,7 @@ use std::str::FromStr; use anyhow::Result; use url::Url; -use distribution_filename::WheelFilename; +use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename}; use pep440_rs::Version; use pep508_rs::VerbatimUrl; use puffin_normalize::PackageName; @@ -58,6 +54,7 @@ pub use crate::file::*; pub use crate::id::*; pub use crate::index_url::*; pub use crate::installed::*; +pub use crate::prioritized_distribution::*; pub use crate::resolution::*; pub use crate::traits::*; @@ -70,6 +67,7 @@ mod file; mod id; mod index_url; mod installed; +mod prioritized_distribution; mod resolution; mod traits; @@ -148,8 +146,7 @@ pub enum SourceDist { /// A built distribution (wheel) that exists in a registry, like `PyPI`. #[derive(Debug, Clone)] pub struct RegistryBuiltDist { - pub name: PackageName, - pub version: Version, + pub filename: WheelFilename, pub file: File, pub index: IndexUrl, } @@ -174,8 +171,7 @@ pub struct PathBuiltDist { /// A source distribution that exists in a registry, like `PyPI`. #[derive(Debug, Clone)] pub struct RegistrySourceDist { - pub name: PackageName, - pub version: Version, + pub filename: SourceDistFilename, pub file: File, pub index: IndexUrl, } @@ -207,24 +203,22 @@ pub struct PathSourceDist { impl Dist { /// Create a [`Dist`] for a registry-based distribution. - pub fn from_registry(name: PackageName, version: Version, file: File, index: IndexUrl) -> Self { - if Path::new(&file.filename) - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("whl")) - { - Self::Built(BuiltDist::Registry(RegistryBuiltDist { - name, - version, - file, - index, - })) - } else { - Self::Source(SourceDist::Registry(RegistrySourceDist { - name, - version, - file, - index, - })) + pub fn from_registry(filename: DistFilename, file: File, index: IndexUrl) -> Self { + match filename { + DistFilename::WheelFilename(filename) => { + Self::Built(BuiltDist::Registry(RegistryBuiltDist { + filename, + file, + index, + })) + } + DistFilename::SourceDistFilename(filename) => { + Self::Source(SourceDist::Registry(RegistrySourceDist { + filename, + file, + index, + })) + } } } @@ -305,6 +299,13 @@ impl Dist { Dist::Source(source) => source.file(), } } + + pub fn version(&self) -> Option<&Version> { + match self { + Dist::Built(wheel) => Some(wheel.version()), + Dist::Source(source_dist) => source_dist.version(), + } + } } impl BuiltDist { @@ -315,6 +316,14 @@ impl BuiltDist { BuiltDist::DirectUrl(_) | BuiltDist::Path(_) => None, } } + + pub fn version(&self) -> &Version { + match self { + BuiltDist::Registry(wheel) => &wheel.filename.version, + BuiltDist::DirectUrl(wheel) => &wheel.filename.version, + BuiltDist::Path(wheel) => &wheel.filename.version, + } + } } impl SourceDist { @@ -326,6 +335,13 @@ impl SourceDist { } } + pub fn version(&self) -> Option<&Version> { + match self { + SourceDist::Registry(source_dist) => Some(&source_dist.filename.version), + SourceDist::DirectUrl(_) | SourceDist::Git(_) | SourceDist::Path(_) => None, + } + } + #[must_use] pub fn with_url(self, url: Url) -> Self { match self { @@ -348,7 +364,7 @@ impl SourceDist { impl Name for RegistryBuiltDist { fn name(&self) -> &PackageName { - &self.name + &self.filename.name } } @@ -366,7 +382,7 @@ impl Name for PathBuiltDist { impl Name for RegistrySourceDist { fn name(&self) -> &PackageName { - &self.name + &self.filename.name } } @@ -420,7 +436,7 @@ impl Name for Dist { impl DistributionMetadata for RegistryBuiltDist { fn version_or_url(&self) -> VersionOrUrl { - VersionOrUrl::Version(&self.version) + VersionOrUrl::Version(&self.filename.version) } } @@ -438,7 +454,7 @@ impl DistributionMetadata for PathBuiltDist { impl DistributionMetadata for RegistrySourceDist { fn version_or_url(&self) -> VersionOrUrl { - VersionOrUrl::Version(&self.version) + VersionOrUrl::Version(&self.filename.version) } } @@ -678,6 +694,22 @@ impl Identifier for Path { } } +impl Identifier for FileLocation { + fn distribution_id(&self) -> DistributionId { + match self { + FileLocation::Url(url) => url.distribution_id(), + FileLocation::Path(path, _) => path.distribution_id(), + } + } + + fn resource_id(&self) -> ResourceId { + match self { + FileLocation::Url(url) => url.resource_id(), + FileLocation::Path(path, _) => path.resource_id(), + } + } +} + impl Identifier for RegistryBuiltDist { fn distribution_id(&self) -> DistributionId { self.file.distribution_id() diff --git a/crates/distribution-types/src/prioritized_distribution.rs b/crates/distribution-types/src/prioritized_distribution.rs new file mode 100644 index 000000000..c2f3174c7 --- /dev/null +++ b/crates/distribution-types/src/prioritized_distribution.rs @@ -0,0 +1,207 @@ +use pep440_rs::VersionSpecifiers; +use platform_tags::TagPriority; +use pypi_types::Hashes; + +use crate::Dist; + +/// Attach its requires-python to a [`Dist`], since downstream needs this information to filter +/// [`PrioritizedDistribution`]. +#[derive(Debug, Clone)] +pub struct DistRequiresPython { + pub dist: Dist, + pub requires_python: Option, +} + +#[derive(Debug, Clone)] +pub struct PrioritizedDistribution { + /// An arbitrary source distribution for the package version. + source: Option, + /// The highest-priority, platform-compatible wheel for the package version. + compatible_wheel: Option<(DistRequiresPython, TagPriority)>, + /// An arbitrary, platform-incompatible wheel for the package version. + incompatible_wheel: Option, + /// The hashes for each distribution. + hashes: Vec, +} + +impl PrioritizedDistribution { + /// Create a new [`PrioritizedDistribution`] from the given wheel distribution. + pub fn from_built( + dist: Dist, + requires_python: Option, + hash: Option, + priority: Option, + ) -> Self { + if let Some(priority) = priority { + Self { + source: None, + compatible_wheel: Some(( + DistRequiresPython { + dist, + + requires_python, + }, + priority, + )), + incompatible_wheel: None, + hashes: hash.map(|hash| vec![hash]).unwrap_or_default(), + } + } else { + Self { + source: None, + compatible_wheel: None, + incompatible_wheel: Some(DistRequiresPython { + dist, + requires_python, + }), + hashes: hash.map(|hash| vec![hash]).unwrap_or_default(), + } + } + } + + /// Create a new [`PrioritizedDistribution`] from the given source distribution. + pub fn from_source( + dist: Dist, + requires_python: Option, + hash: Option, + ) -> Self { + Self { + source: Some(DistRequiresPython { + dist, + requires_python, + }), + compatible_wheel: None, + incompatible_wheel: None, + hashes: hash.map(|hash| vec![hash]).unwrap_or_default(), + } + } + + /// Insert the given built distribution into the [`PrioritizedDistribution`]. + pub fn insert_built( + &mut self, + dist: Dist, + requires_python: Option, + hash: Option, + priority: Option, + ) { + // Prefer the highest-priority, platform-compatible wheel. + if let Some(priority) = priority { + if let Some((.., existing_priority)) = &self.compatible_wheel { + if priority > *existing_priority { + self.compatible_wheel = Some(( + DistRequiresPython { + dist, + requires_python, + }, + priority, + )); + } + } else { + self.compatible_wheel = Some(( + DistRequiresPython { + dist, + requires_python, + }, + priority, + )); + } + } else if self.incompatible_wheel.is_none() { + self.incompatible_wheel = Some(DistRequiresPython { + dist, + requires_python, + }); + } + + if let Some(hash) = hash { + self.hashes.push(hash); + } + } + + /// Insert the given source distribution into the [`PrioritizedDistribution`]. + pub fn insert_source( + &mut self, + dist: Dist, + requires_python: Option, + hash: Option, + ) { + if self.source.is_none() { + self.source = Some(DistRequiresPython { + dist, + requires_python, + }); + } + + if let Some(hash) = hash { + self.hashes.push(hash); + } + } + + /// Return the highest-priority distribution for the package version, if any. + pub fn get(&self) -> Option { + match ( + &self.compatible_wheel, + &self.source, + &self.incompatible_wheel, + ) { + // Prefer the highest-priority, platform-compatible wheel. + (Some((wheel, tag_priority)), _, _) => { + Some(ResolvableDist::CompatibleWheel(wheel, *tag_priority)) + } + // If we have a compatible source distribution and an incompatible wheel, return the + // wheel. We assume that all distributions have the same metadata for a given package + // version. If a compatible source distribution exists, we assume we can build it, but + // using the wheel is faster. + (_, Some(source_dist), Some(wheel)) => { + Some(ResolvableDist::IncompatibleWheel { source_dist, wheel }) + } + // Otherwise, if we have a source distribution, return it. + (_, Some(source_dist), _) => Some(ResolvableDist::SourceDist(source_dist)), + _ => None, + } + } + + /// Return the hashes for each distribution. + pub fn hashes(&self) -> &[Hashes] { + &self.hashes + } +} + +#[derive(Debug, Clone)] +pub enum ResolvableDist<'a> { + /// The distribution should be resolved and installed using a source distribution. + SourceDist(&'a DistRequiresPython), + /// The distribution should be resolved and installed using a wheel distribution. + CompatibleWheel(&'a DistRequiresPython, TagPriority), + /// The distribution should be resolved using an incompatible wheel distribution, but + /// installed using a source distribution. + IncompatibleWheel { + source_dist: &'a DistRequiresPython, + wheel: &'a DistRequiresPython, + }, +} + +impl<'a> ResolvableDist<'a> { + /// Return the [`DistRequiresPython`] to use during resolution. + pub fn resolve(&self) -> &DistRequiresPython { + match *self { + ResolvableDist::SourceDist(sdist) => sdist, + ResolvableDist::CompatibleWheel(wheel, _) => wheel, + ResolvableDist::IncompatibleWheel { + source_dist: _, + wheel, + } => wheel, + } + } + + /// Return the [`DistRequiresPython`] to use during installation. + pub fn install(&self) -> &DistRequiresPython { + match *self { + ResolvableDist::SourceDist(sdist) => sdist, + ResolvableDist::CompatibleWheel(wheel, _) => wheel, + ResolvableDist::IncompatibleWheel { + source_dist, + wheel: _, + } => source_dist, + } + } +} diff --git a/crates/distribution-types/src/resolution.rs b/crates/distribution-types/src/resolution.rs index 6557e5d5e..b4f14df12 100644 --- a/crates/distribution-types/src/resolution.rs +++ b/crates/distribution-types/src/resolution.rs @@ -65,11 +65,11 @@ impl From for Requirement { fn from(dist: Dist) -> Self { match dist { Dist::Built(BuiltDist::Registry(wheel)) => Requirement { - name: wheel.name, + name: wheel.filename.name, extras: None, version_or_url: Some(pep508_rs::VersionOrUrl::VersionSpecifier( pep440_rs::VersionSpecifiers::from( - pep440_rs::VersionSpecifier::equals_version(wheel.version), + pep440_rs::VersionSpecifier::equals_version(wheel.filename.version), ), )), marker: None, @@ -87,11 +87,11 @@ impl From for Requirement { marker: None, }, Dist::Source(SourceDist::Registry(sdist)) => Requirement { - name: sdist.name, + name: sdist.filename.name, extras: None, version_or_url: Some(pep508_rs::VersionOrUrl::VersionSpecifier( pep440_rs::VersionSpecifiers::from( - pep440_rs::VersionSpecifier::equals_version(sdist.version), + pep440_rs::VersionSpecifier::equals_version(sdist.filename.version), ), )), marker: None, diff --git a/crates/pep508-rs/src/verbatim_url.rs b/crates/pep508-rs/src/verbatim_url.rs index 6bbe48356..eb1736ffc 100644 --- a/crates/pep508-rs/src/verbatim_url.rs +++ b/crates/pep508-rs/src/verbatim_url.rs @@ -4,11 +4,13 @@ use std::path::Path; use once_cell::sync::Lazy; use regex::Regex; +use serde::{Deserialize, Serialize}; use url::Url; /// A wrapper around [`Url`] that preserves the original string. #[derive(Debug, Clone, Eq, derivative::Derivative)] #[derivative(PartialEq, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct VerbatimUrl { /// The parsed URL. url: Url, diff --git a/crates/puffin-cli/src/commands/pip_compile.rs b/crates/puffin-cli/src/commands/pip_compile.rs index a5bbf0986..c88a9c10e 100644 --- a/crates/puffin-cli/src/commands/pip_compile.rs +++ b/crates/puffin-cli/src/commands/pip_compile.rs @@ -13,7 +13,7 @@ use owo_colors::OwoColorize; use tempfile::tempdir_in; use tracing::debug; -use distribution_types::{IndexUrls, LocalEditable}; +use distribution_types::{IndexLocations, LocalEditable}; use pep508_rs::Requirement; use platform_host::Platform; use platform_tags::Tags; @@ -48,7 +48,7 @@ pub(crate) async fn pip_compile( prerelease_mode: PreReleaseMode, upgrade_mode: UpgradeMode, generate_hashes: bool, - index_urls: IndexUrls, + index_locations: IndexLocations, setup_py: SetupPyStrategy, no_build: bool, python_version: Option, @@ -144,7 +144,7 @@ pub(crate) async fn pip_compile( // Instantiate a client. let client = RegistryClientBuilder::new(cache.clone()) - .index_urls(index_urls.clone()) + .index_locations(index_locations.clone()) .build(); let options = ResolutionOptions::new(resolution_mode, prerelease_mode, exclude_newer); @@ -152,7 +152,7 @@ pub(crate) async fn pip_compile( &client, &cache, &interpreter, - &index_urls, + &index_locations, interpreter.sys_executable().to_path_buf(), setup_py, no_build, @@ -228,7 +228,7 @@ pub(crate) async fn pip_compile( &tags, &client, &build_dispatch, - ) + )? .with_reporter(ResolverReporter::from(printer)); let resolution = match resolver.resolve().await { Err(puffin_resolver::ResolveError::NoSolution(err)) => { diff --git a/crates/puffin-cli/src/commands/pip_install.rs b/crates/puffin-cli/src/commands/pip_install.rs index a45f9dff3..03c943c74 100644 --- a/crates/puffin-cli/src/commands/pip_install.rs +++ b/crates/puffin-cli/src/commands/pip_install.rs @@ -11,7 +11,7 @@ use tempfile::tempdir_in; use tracing::debug; use distribution_types::{ - IndexUrls, InstalledMetadata, LocalDist, LocalEditable, Name, Resolution, + IndexLocations, InstalledMetadata, LocalDist, LocalEditable, Name, Resolution, }; use install_wheel_rs::linker::LinkMode; use pep508_rs::{MarkerEnvironment, Requirement}; @@ -45,7 +45,7 @@ pub(crate) async fn pip_install( extras: &ExtrasSpecification<'_>, resolution_mode: ResolutionMode, prerelease_mode: PreReleaseMode, - index_urls: IndexUrls, + index_locations: IndexLocations, reinstall: &Reinstall, link_mode: LinkMode, setup_py: SetupPyStrategy, @@ -134,7 +134,7 @@ pub(crate) async fn pip_install( // Instantiate a client. let client = RegistryClientBuilder::new(cache.clone()) - .index_urls(index_urls.clone()) + .index_locations(index_locations.clone()) .build(); let options = ResolutionOptions::new(resolution_mode, prerelease_mode, exclude_newer); @@ -143,7 +143,7 @@ pub(crate) async fn pip_install( &client, &cache, &interpreter, - &index_urls, + &index_locations, venv.python_executable(), setup_py, no_build, @@ -209,7 +209,7 @@ pub(crate) async fn pip_install( site_packages, reinstall, link_mode, - &index_urls, + &index_locations, tags, &client, &build_dispatch, @@ -379,7 +379,7 @@ async fn resolve( tags, client, build_dispatch, - ) + )? .with_reporter(ResolverReporter::from(printer)); let resolution = resolver.resolve().await?; @@ -406,7 +406,7 @@ async fn install( site_packages: SitePackages<'_>, reinstall: &Reinstall, link_mode: LinkMode, - index_urls: &IndexUrls, + index_urls: &IndexLocations, tags: &Tags, client: &RegistryClient, build_dispatch: &BuildDispatch<'_>, @@ -603,6 +603,9 @@ enum Error { #[error(transparent)] Resolve(#[from] puffin_resolver::ResolveError), + #[error(transparent)] + Client(#[from] puffin_client::Error), + #[error(transparent)] Platform(#[from] platform_host::PlatformError), diff --git a/crates/puffin-cli/src/commands/pip_sync.rs b/crates/puffin-cli/src/commands/pip_sync.rs index ac8907bbd..bd0f8b400 100644 --- a/crates/puffin-cli/src/commands/pip_sync.rs +++ b/crates/puffin-cli/src/commands/pip_sync.rs @@ -5,7 +5,7 @@ use itertools::Itertools; use owo_colors::OwoColorize; use tracing::debug; -use distribution_types::{IndexUrls, InstalledMetadata, LocalDist, LocalEditable, Name}; +use distribution_types::{IndexLocations, InstalledMetadata, LocalDist, LocalEditable, Name}; use install_wheel_rs::linker::LinkMode; use platform_host::Platform; use platform_tags::Tags; @@ -29,7 +29,7 @@ pub(crate) async fn pip_sync( sources: &[RequirementsSource], reinstall: &Reinstall, link_mode: LinkMode, - index_urls: IndexUrls, + index_locations: IndexLocations, setup_py: SetupPyStrategy, no_build: bool, strict: bool, @@ -60,7 +60,7 @@ pub(crate) async fn pip_sync( // Prep the registry client. let client = RegistryClientBuilder::new(cache.clone()) - .index_urls(index_urls.clone()) + .index_locations(index_locations.clone()) .build(); // Prep the build context. @@ -68,7 +68,7 @@ pub(crate) async fn pip_sync( &client, &cache, venv.interpreter(), - &index_urls, + &index_locations, venv.python_executable(), setup_py, no_build, @@ -104,7 +104,7 @@ pub(crate) async fn pip_sync( resolved_editables.editables, site_packages, reinstall, - &index_urls, + &index_locations, &cache, &venv, tags, @@ -130,7 +130,7 @@ pub(crate) async fn pip_sync( // Instantiate a client. let client = RegistryClientBuilder::new(cache.clone()) - .index_urls(index_urls.clone()) + .index_locations(index_locations.clone()) .build(); // Resolve any registry-based requirements. diff --git a/crates/puffin-cli/src/commands/venv.rs b/crates/puffin-cli/src/commands/venv.rs index 957d78cc4..be02dc333 100644 --- a/crates/puffin-cli/src/commands/venv.rs +++ b/crates/puffin-cli/src/commands/venv.rs @@ -8,7 +8,7 @@ use miette::{Diagnostic, IntoDiagnostic}; use owo_colors::OwoColorize; use thiserror::Error; -use distribution_types::{DistributionMetadata, IndexUrls, Name}; +use distribution_types::{DistributionMetadata, IndexLocations, Name}; use pep508_rs::Requirement; use platform_host::Platform; use puffin_cache::Cache; @@ -25,12 +25,12 @@ use crate::printer::Printer; pub(crate) async fn venv( path: &Path, base_python: Option<&Path>, - index_urls: &IndexUrls, + index_locations: &IndexLocations, seed: bool, cache: &Cache, printer: Printer, ) -> Result { - match venv_impl(path, base_python, index_urls, seed, cache, printer).await { + match venv_impl(path, base_python, index_locations, seed, cache, printer).await { Ok(status) => Ok(status), Err(err) => { #[allow(clippy::print_stderr)] @@ -69,7 +69,7 @@ enum VenvError { async fn venv_impl( path: &Path, base_python: Option<&Path>, - index_urls: &IndexUrls, + index_locations: &IndexLocations, seed: bool, cache: &Cache, mut printer: Printer, @@ -122,7 +122,7 @@ async fn venv_impl( &client, cache, venv.interpreter(), - index_urls, + index_locations, venv.python_executable(), SetupPyStrategy::default(), true, diff --git a/crates/puffin-cli/src/main.rs b/crates/puffin-cli/src/main.rs index b5d0253a1..02b741f51 100644 --- a/crates/puffin-cli/src/main.rs +++ b/crates/puffin-cli/src/main.rs @@ -8,7 +8,7 @@ use chrono::{DateTime, Days, NaiveDate, NaiveTime, Utc}; use clap::{Args, Parser, Subcommand}; use owo_colors::OwoColorize; -use distribution_types::{IndexUrl, IndexUrls}; +use distribution_types::{FlatIndexLocation, IndexLocations, IndexUrl}; use puffin_cache::{Cache, CacheArgs}; use puffin_installer::Reinstall; use puffin_interpreter::PythonVersion; @@ -159,6 +159,15 @@ struct PipCompileArgs { #[clap(long)] extra_index_url: Vec, + /// Locations to search for candidate distributions, beyond those found in the indexes. + /// + /// If a path, the target must be a directory that contains package as wheel files (`.whl`) or + /// source distributions (`.tar.gz` or `.zip`) at the top level. + /// + /// If a URL, the page must contain a flat list of links to package files. + #[clap(long)] + find_links: Vec, + /// Ignore the package index, instead relying on local archives and caches. #[clap(long, conflicts_with = "index_url", conflicts_with = "extra_index_url")] no_index: bool, @@ -234,6 +243,15 @@ struct PipSyncArgs { #[clap(long)] extra_index_url: Vec, + /// Locations to search for candidate distributions, beyond those found in the indexes. + /// + /// If a path, the target must be a directory that contains package as wheel files (`.whl`) or + /// source distributions (`.tar.gz` or `.zip`) at the top level. + /// + /// If a URL, the page must contain a flat list of links to package files. + #[clap(long)] + find_links: Vec, + /// Ignore the package index, instead relying on local archives and caches. #[clap(long, conflicts_with = "index_url", conflicts_with = "extra_index_url")] no_index: bool, @@ -335,6 +353,15 @@ struct PipInstallArgs { #[clap(long)] extra_index_url: Vec, + /// Locations to search for candidate distributions, beyond those found in the indexes. + /// + /// If a path, the target must be a directory that contains package as wheel files (`.whl`) or + /// source distributions (`.tar.gz` or `.zip`) at the top level. + /// + /// If a URL, the page must contain a flat list of links to package files. + #[clap(long)] + find_links: Vec, + /// Ignore the package index, instead relying on local archives and caches. #[clap(long, conflicts_with = "index_url", conflicts_with = "extra_index_url")] no_index: bool, @@ -497,8 +524,12 @@ async fn inner() -> Result { .into_iter() .map(RequirementsSource::from) .collect::>(); - let index_urls = - IndexUrls::from_args(args.index_url, args.extra_index_url, args.no_index); + let index_urls = IndexLocations::from_args( + args.index_url, + args.extra_index_url, + args.find_links, + args.no_index, + ); let extras = if args.all_extras { ExtrasSpecification::All } else if args.extra.is_empty() { @@ -531,8 +562,12 @@ async fn inner() -> Result { .await } Commands::PipSync(args) => { - let index_urls = - IndexUrls::from_args(args.index_url, args.extra_index_url, args.no_index); + let index_urls = IndexLocations::from_args( + args.index_url, + args.extra_index_url, + args.find_links, + args.no_index, + ); let sources = args .src_file .into_iter() @@ -574,8 +609,12 @@ async fn inner() -> Result { .into_iter() .map(RequirementsSource::from) .collect::>(); - let index_urls = - IndexUrls::from_args(args.index_url, args.extra_index_url, args.no_index); + let index_urls = IndexLocations::from_args( + args.index_url, + args.extra_index_url, + args.find_links, + args.no_index, + ); let extras = if args.all_extras { ExtrasSpecification::All } else if args.extra.is_empty() { @@ -620,12 +659,17 @@ async fn inner() -> Result { Commands::Clean(args) => commands::clean(&cache, &args.package, printer), Commands::PipFreeze(args) => commands::freeze(&cache, args.strict, printer), Commands::Venv(args) => { - let index_urls = - IndexUrls::from_args(args.index_url, args.extra_index_url, args.no_index); + let index_locations = IndexLocations::from_args( + args.index_url, + args.extra_index_url, + // No find links for the venv subcommand, to keep things simple + Vec::new(), + args.no_index, + ); commands::venv( &args.name, args.python.as_deref(), - &index_urls, + &index_locations, args.seed, &cache, printer, diff --git a/crates/puffin-cli/tests/pip_compile.rs b/crates/puffin-cli/tests/pip_compile.rs index 499046691..a87a29bc0 100644 --- a/crates/puffin-cli/tests/pip_compile.rs +++ b/crates/puffin-cli/tests/pip_compile.rs @@ -3035,3 +3035,56 @@ fn generate_hashes() -> Result<()> { Ok(()) } + +/// Make sure find links are correctly resolved and reported +#[test] +fn find_links() -> Result<()> { + let temp_dir = TempDir::new()?; + let cache_dir = TempDir::new()?; + let venv = create_venv_py312(&temp_dir, &cache_dir); + + let requirements_in = temp_dir.child("requirements.in"); + requirements_in.write_str(indoc! {r" + tqdm + numpy + werkzeug @ https://files.pythonhosted.org/packages/c3/fc/254c3e9b5feb89ff5b9076a23218dafbc99c96ac5941e900b71206e6313b/werkzeug-3.0.1-py3-none-any.whl + "})?; + + let project_root = fs_err::canonicalize(std::env::current_dir()?.join("../.."))?; + let project_root_string = project_root.display().to_string(); + let filters: Vec<_> = iter::once((project_root_string.as_str(), "[PROJECT_ROOT]")) + .chain(INSTA_FILTERS.to_vec()) + .collect(); + + insta::with_settings!({ + filters => filters + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip-compile") + .arg("requirements.in") + .arg("--find-links") + .arg(project_root.join("scripts/wheels/")) + .arg("--cache-dir") + .arg(cache_dir.path()) + .arg("--exclude-newer") + .arg(EXCLUDE_NEWER) + .env("VIRTUAL_ENV", venv.as_os_str()) + .current_dir(&temp_dir), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by Puffin v0.0.1 via the following command: + # puffin pip-compile requirements.in --find-links [PROJECT_ROOT]/scripts/wheels/ --cache-dir [CACHE_DIR] + markupsafe==2.1.3 + # via werkzeug + numpy==1.26.2 + tqdm==1000.0.0 + werkzeug @ https://files.pythonhosted.org/packages/c3/fc/254c3e9b5feb89ff5b9076a23218dafbc99c96ac5941e900b71206e6313b/werkzeug-3.0.1-py3-none-any.whl + + ----- stderr ----- + Resolved 4 packages in [TIME] + "###); + }); + + Ok(()) +} diff --git a/crates/puffin-client/Cargo.toml b/crates/puffin-client/Cargo.toml index 4c84a26d4..41f4e8188 100644 --- a/crates/puffin-client/Cargo.toml +++ b/crates/puffin-client/Cargo.toml @@ -9,6 +9,8 @@ distribution-filename = { path = "../distribution-filename", features = ["serde" distribution-types = { path = "../distribution-types" } install-wheel-rs = { path = "../install-wheel-rs" } pep440_rs = { path = "../pep440-rs" } +pep508_rs = { path = "../pep508-rs" } +platform-tags = { path = "../platform-tags" } puffin-cache = { path = "../puffin-cache" } puffin-fs = { path = "../puffin-fs" } puffin-normalize = { path = "../puffin-normalize" } @@ -16,6 +18,7 @@ pypi-types = { path = "../pypi-types" } async_http_range_reader = { workspace = true } async_zip = { workspace = true, features = ["tokio"] } +chrono = { workspace = true } fs-err = { workspace = true, features = ["tokio"] } futures = { workspace = true } html-escape = { workspace = true } @@ -25,6 +28,7 @@ reqwest = { workspace = true } reqwest-middleware = { workspace = true } reqwest-retry = { workspace = true } rmp-serde = { workspace = true } +rustc-hash = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } @@ -37,8 +41,6 @@ tracing = { workspace = true } url = { workspace = true } [dev-dependencies] -pep508_rs = { path = "../pep508-rs" } - anyhow = { workspace = true } insta = { version = "1.34.0" } tokio = { workspace = true, features = ["fs", "macros"] } diff --git a/crates/puffin-client/src/error.rs b/crates/puffin-client/src/error.rs index 78b8e0594..cb61e310c 100644 --- a/crates/puffin-client/src/error.rs +++ b/crates/puffin-client/src/error.rs @@ -31,7 +31,7 @@ pub enum Error { /// The metadata file could not be parsed. #[error("Couldn't parse metadata of {0} from {1}")] - MetadataParseError(WheelFilename, String, #[source] pypi_types::Error), + MetadataParseError(WheelFilename, String, #[source] Box), /// The metadata file was not found in the registry. #[error("File `{0}` was not found in the registry at {1}.")] @@ -95,6 +95,9 @@ pub enum Error { #[error("Unsupported `Content-Type` \"{1}\" for {0}. Expected JSON or HTML.")] UnsupportedMediaType(Url, String), + + #[error("Failed to read find links directory")] + FindLinks(#[source] io::Error), } impl Error { diff --git a/crates/puffin-client/src/flat_index.rs b/crates/puffin-client/src/flat_index.rs new file mode 100644 index 000000000..9d947ef53 --- /dev/null +++ b/crates/puffin-client/src/flat_index.rs @@ -0,0 +1,104 @@ +use std::collections::btree_map::Entry; +use std::collections::BTreeMap; +use std::path::PathBuf; + +use rustc_hash::FxHashMap; +use tracing::instrument; + +use distribution_filename::DistFilename; +use distribution_types::{ + BuiltDist, Dist, File, FileLocation, IndexUrl, PrioritizedDistribution, RegistryBuiltDist, + RegistrySourceDist, SourceDist, +}; +use pep440_rs::Version; +use pep508_rs::VerbatimUrl; +use platform_tags::Tags; +use puffin_normalize::PackageName; +use pypi_types::Hashes; + +#[derive(Debug, Clone)] +pub struct FlatIndex + From + Ord>( + pub BTreeMap, +); + +impl + From + Ord> Default for FlatIndex { + fn default() -> Self { + Self(BTreeMap::default()) + } +} + +impl + From + Ord> FlatIndex { + /// Collect all the files from `--find-links` into a override hashmap we can pass into version map creation. + #[instrument(skip_all)] + pub fn from_dists( + dists: Vec<(DistFilename, PathBuf)>, + tags: &Tags, + ) -> FxHashMap { + // If we have packages of the same name from find links, gives them priority, otherwise start empty + let mut flat_index: FxHashMap = FxHashMap::default(); + + // Collect compatible distributions. + for (filename, path) in dists { + let version_map = flat_index.entry(filename.name().clone()).or_default(); + + let url = VerbatimUrl::from_path(&path, path.display().to_string()) + .expect("Find link paths must be absolute"); + let file = File { + dist_info_metadata: None, + filename: filename.to_string(), + hashes: Hashes { sha256: None }, + requires_python: None, + size: None, + upload_time: None, + url: FileLocation::Path(path.to_path_buf(), url), + yanked: None, + }; + + // No `requires-python` here: for source distributions, we don't have that information; + // for wheels, we read it lazily only when selected. + match filename { + DistFilename::WheelFilename(filename) => { + let priority = filename.compatibility(tags); + let version = filename.version.clone(); + + let dist = Dist::Built(BuiltDist::Registry(RegistryBuiltDist { + filename, + file, + index: IndexUrl::Pypi, + })); + match version_map.0.entry(version.into()) { + Entry::Occupied(mut entry) => { + entry.get_mut().insert_built(dist, None, None, priority); + } + Entry::Vacant(entry) => { + entry.insert(PrioritizedDistribution::from_built( + dist, None, None, priority, + )); + } + } + } + DistFilename::SourceDistFilename(filename) => { + let dist = Dist::Source(SourceDist::Registry(RegistrySourceDist { + filename: filename.clone(), + file, + index: IndexUrl::Pypi, + })); + match version_map.0.entry(filename.version.clone().into()) { + Entry::Occupied(mut entry) => { + entry.get_mut().insert_source(dist, None, None); + } + Entry::Vacant(entry) => { + entry.insert(PrioritizedDistribution::from_source(dist, None, None)); + } + } + } + } + } + + flat_index + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } +} diff --git a/crates/puffin-client/src/lib.rs b/crates/puffin-client/src/lib.rs index ea925a8fb..580aa8239 100644 --- a/crates/puffin-client/src/lib.rs +++ b/crates/puffin-client/src/lib.rs @@ -1,11 +1,13 @@ pub use cached_client::{CachedClient, CachedClientError, DataWithCachePolicy}; pub use error::Error; +pub use flat_index::FlatIndex; pub use registry_client::{ read_metadata_async, RegistryClient, RegistryClientBuilder, SimpleMetadata, VersionFiles, }; mod cached_client; mod error; +mod flat_index; mod html; mod registry_client; mod remote_metadata; diff --git a/crates/puffin-client/src/registry_client.rs b/crates/puffin-client/src/registry_client.rs index 596fe1b96..53e1b4331 100644 --- a/crates/puffin-client/src/registry_client.rs +++ b/crates/puffin-client/src/registry_client.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use std::fmt::Debug; -use std::path::Path; +use std::io; +use std::path::{Path, PathBuf}; use std::str::FromStr; use async_http_range_reader::{AsyncHttpRangeReader, AsyncHttpRangeReaderError}; @@ -17,7 +18,9 @@ use tracing::{debug, info_span, instrument, trace, warn, Instrument}; use url::Url; use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename}; -use distribution_types::{BuiltDist, File, IndexUrl, IndexUrls, Name}; +use distribution_types::{ + BuiltDist, File, FileLocation, FlatIndexLocation, IndexLocations, IndexUrl, Name, +}; use install_wheel_rs::find_dist_info; use pep440_rs::Version; use puffin_cache::{Cache, CacheBucket, WheelCache}; @@ -31,7 +34,7 @@ use crate::{CachedClient, CachedClientError, Error}; /// A builder for an [`RegistryClient`]. #[derive(Debug, Clone)] pub struct RegistryClientBuilder { - index_urls: IndexUrls, + index_locations: IndexLocations, retries: u32, cache: Cache, } @@ -39,7 +42,7 @@ pub struct RegistryClientBuilder { impl RegistryClientBuilder { pub fn new(cache: Cache) -> Self { Self { - index_urls: IndexUrls::default(), + index_locations: IndexLocations::default(), cache, retries: 3, } @@ -48,8 +51,8 @@ impl RegistryClientBuilder { impl RegistryClientBuilder { #[must_use] - pub fn index_urls(mut self, index_urls: IndexUrls) -> Self { - self.index_urls = index_urls; + pub fn index_locations(mut self, index_urls: IndexLocations) -> Self { + self.index_locations = index_urls; self } @@ -84,7 +87,7 @@ impl RegistryClientBuilder { let client = CachedClient::new(uncached_client.clone()); RegistryClient { - index_urls: self.index_urls, + index_locations: self.index_locations, client_raw: client_raw.clone(), cache: self.cache, client, @@ -96,7 +99,7 @@ impl RegistryClientBuilder { #[derive(Debug, Clone)] pub struct RegistryClient { /// The index URLs to use for fetching packages. - index_urls: IndexUrls, + index_locations: IndexLocations, /// The underlying HTTP client. client: CachedClient, /// Don't use this client, it only exists because `async_http_range_reader` needs @@ -112,6 +115,68 @@ impl RegistryClient { &self.client } + /// Read the directories and flat remote indexes from `--find-links`. + #[allow(clippy::result_large_err)] + pub fn flat_index(&self) -> Result, Error> { + let mut dists = Vec::new(); + for flat_index in self.index_locations.flat_indexes() { + match flat_index { + FlatIndexLocation::Path(path) => { + dists.extend(Self::read_flat_index_dir(path).map_err(Error::FindLinks)?); + } + FlatIndexLocation::Url(_) => { + warn!("TODO(konstin): No yet implemented: Find links urls"); + } + } + } + Ok(dists) + } + + /// Read a list of [`DistFilename`] entries from a `--find-links` directory.. + fn read_flat_index_dir(path: &PathBuf) -> Result, io::Error> { + // Absolute paths are required for the URL conversion. + let path = fs_err::canonicalize(path)?; + + let mut dists = Vec::new(); + for entry in fs_err::read_dir(&path)? { + let entry = entry?; + let metadata = entry.metadata()?; + if !metadata.is_file() { + continue; + } + + let Ok(filename) = entry.file_name().into_string() else { + warn!( + "Skipping non-UTF-8 filename in `--find-links` directory: {}", + entry.file_name().to_string_lossy() + ); + continue; + }; + let Some(filename) = DistFilename::try_from_normalized_filename(&filename) else { + debug!( + "Ignoring `--find-links` entry (expected a wheel or source distribution filename): {}", + entry.path().display() + ); + continue; + }; + let path = entry.path().to_path_buf(); + dists.push((filename, path)); + } + if dists.is_empty() { + warn!( + "No packages found in `--find-links` directory: {}", + path.display() + ); + } else { + debug!( + "Found {} packages in `--find-links` directory: {}", + dists.len(), + path.display() + ); + } + Ok(dists) + } + /// Fetch a package from the `PyPI` simple API. /// /// "simple" here refers to [PEP 503 – Simple Repository API](https://peps.python.org/pep-0503/) @@ -122,77 +187,13 @@ impl RegistryClient { &self, package_name: &PackageName, ) -> Result<(IndexUrl, SimpleMetadata), Error> { - if self.index_urls.no_index() { + if self.index_locations.no_index() { return Err(Error::NoIndex(package_name.as_ref().to_string())); } - for index in &self.index_urls { - // Format the URL for PyPI. - let mut url: Url = index.clone().into(); - url.path_segments_mut() - .unwrap() - .pop_if_empty() - .push(package_name.as_ref()); + for index in self.index_locations.indexes() { + let result = self.simple_single_index(package_name, index).await?; - trace!("Fetching metadata for {package_name} from {url}"); - - let cache_entry = self.cache.entry( - CacheBucket::Simple, - Path::new(&match index { - IndexUrl::Pypi => "pypi".to_string(), - IndexUrl::Url(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)), - }), - format!("{package_name}.msgpack"), - ); - - let simple_request = self - .client - .uncached() - .get(url.clone()) - .header("Accept-Encoding", "gzip") - .header("Accept", MediaType::accepts()) - .build()?; - let parse_simple_response = |response: Response| { - async { - let content_type = response - .headers() - .get("content-type") - .ok_or_else(|| Error::MissingContentType(url.clone()))?; - let content_type = content_type - .to_str() - .map_err(|err| Error::InvalidContentTypeHeader(url.clone(), err))?; - let media_type = content_type.split(';').next().unwrap_or(content_type); - let media_type = MediaType::from_str(media_type).ok_or_else(|| { - Error::UnsupportedMediaType(url.clone(), media_type.to_string()) - })?; - - match media_type { - MediaType::Json => { - let bytes = response.bytes().await?; - let data: SimpleJson = serde_json::from_slice(bytes.as_ref()) - .map_err(|err| Error::from_json_err(err, url.clone()))?; - let base = BaseUrl::from(url.clone()); - let metadata = - SimpleMetadata::from_files(data.files, package_name, &base); - Ok(metadata) - } - MediaType::Html => { - let text = response.text().await?; - let SimpleHtml { base, files } = SimpleHtml::parse(&text, &url) - .map_err(|err| Error::from_html_err(err, url.clone()))?; - let metadata = SimpleMetadata::from_files(files, package_name, &base); - Ok(metadata) - } - } - } - .instrument(info_span!("parse_simple_api", package = %package_name)) - }; - let result = self - .client - .get_cached_with_callback(simple_request, &cache_entry, parse_simple_response) - .await; - - // Fetch from the index. return match result { Ok(metadata) => Ok((index.clone(), metadata)), Err(CachedClientError::Client(Error::RequestError(err))) => { @@ -208,6 +209,77 @@ impl RegistryClient { Err(Error::PackageNotFound(package_name.to_string())) } + async fn simple_single_index( + &self, + package_name: &PackageName, + index: &IndexUrl, + ) -> Result>, Error> { + // Format the URL for PyPI. + let mut url: Url = index.clone().into(); + url.path_segments_mut() + .unwrap() + .pop_if_empty() + .push(package_name.as_ref()); + + trace!("Fetching metadata for {package_name} from {url}"); + + let cache_entry = self.cache.entry( + CacheBucket::Simple, + Path::new(&match index { + IndexUrl::Pypi => "pypi".to_string(), + IndexUrl::Url(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)), + }), + format!("{package_name}.msgpack"), + ); + + let simple_request = self + .client + .uncached() + .get(url.clone()) + .header("Accept-Encoding", "gzip") + .header("Accept", MediaType::accepts()) + .build()?; + let parse_simple_response = |response: Response| { + async { + let content_type = response + .headers() + .get("content-type") + .ok_or_else(|| Error::MissingContentType(url.clone()))?; + let content_type = content_type + .to_str() + .map_err(|err| Error::InvalidContentTypeHeader(url.clone(), err))?; + let media_type = content_type.split(';').next().unwrap_or(content_type); + let media_type = MediaType::from_str(media_type).ok_or_else(|| { + Error::UnsupportedMediaType(url.clone(), media_type.to_string()) + })?; + + match media_type { + MediaType::Json => { + let bytes = response.bytes().await?; + let data: SimpleJson = serde_json::from_slice(bytes.as_ref()) + .map_err(|err| Error::from_json_err(err, url.clone()))?; + let base = BaseUrl::from(url.clone()); + let metadata = SimpleMetadata::from_files(data.files, package_name, &base); + Ok(metadata) + } + MediaType::Html => { + let text = response.text().await?; + let SimpleHtml { base, files } = SimpleHtml::parse(&text, &url) + .map_err(|err| Error::from_html_err(err, url.clone()))?; + let metadata = SimpleMetadata::from_files(files, package_name, &base); + Ok(metadata) + } + } + } + .instrument(info_span!("parse_simple_api", package = %package_name)) + }; + let result = self + .client + .get_cached_with_callback(simple_request, &cache_entry, parse_simple_response) + .await; + Ok(result) + } + /// Fetch the metadata for a remote wheel file. /// /// For a remote wheel, we try the following ways to fetch the metadata: @@ -217,10 +289,16 @@ impl RegistryClient { #[instrument(skip(self))] pub async fn wheel_metadata(&self, built_dist: &BuiltDist) -> Result { let metadata = match &built_dist { - BuiltDist::Registry(wheel) => { - self.wheel_metadata_registry(&wheel.index, &wheel.file) - .await? - } + BuiltDist::Registry(wheel) => match &wheel.file.url { + FileLocation::Url(url) => { + self.wheel_metadata_registry(&wheel.index, &wheel.file, url) + .await? + } + FileLocation::Path(path, _url) => { + let reader = fs_err::tokio::File::open(&path).await?; + read_metadata_async(&wheel.filename, built_dist.to_string(), reader).await? + } + }, BuiltDist::DirectUrl(wheel) => { self.wheel_metadata_no_pep658( &wheel.filename, @@ -250,8 +328,9 @@ impl RegistryClient { &self, index: &IndexUrl, file: &File, + url: &Url, ) -> Result { - if self.index_urls.no_index() { + if self.index_locations.no_index() { return Err(Error::NoIndex(file.filename.clone())); } @@ -262,7 +341,7 @@ impl RegistryClient { .as_ref() .is_some_and(pypi_types::DistInfoMetadata::is_available) { - let url = Url::parse(&format!("{}.metadata", file.url))?; + let url = Url::parse(&format!("{}.metadata", url))?; let cache_entry = self.cache.entry( CacheBucket::Wheels, @@ -275,7 +354,9 @@ impl RegistryClient { info_span!("parse_metadata21") .in_scope(|| Metadata21::parse(bytes.as_ref())) - .map_err(|err| Error::MetadataParseError(filename, url.to_string(), err)) + .map_err(|err| { + Error::MetadataParseError(filename, url.to_string(), Box::new(err)) + }) }; let req = self.client.uncached().get(url.clone()).build()?; Ok(self @@ -286,7 +367,7 @@ impl RegistryClient { // If we lack PEP 658 support, try using HTTP range requests to read only the // `.dist-info/METADATA` file from the zip, and if that also fails, download the whole wheel // into the cache and read from there - self.wheel_metadata_no_pep658(&filename, &file.url, WheelCache::Index(index)) + self.wheel_metadata_no_pep658(&filename, url, WheelCache::Index(index)) .await } } @@ -298,7 +379,7 @@ impl RegistryClient { url: &'data Url, cache_shard: WheelCache<'data>, ) -> Result { - if self.index_urls.no_index() { + if self.index_locations.no_index() { return Err(Error::NoIndex(url.to_string())); } @@ -317,7 +398,7 @@ impl RegistryClient { trace!("Getting metadata for {filename} by range request"); let text = wheel_metadata_from_remote_zip(filename, &mut reader).await?; let metadata = Metadata21::parse(text.as_bytes()).map_err(|err| { - Error::MetadataParseError(filename.clone(), url.to_string(), err) + Error::MetadataParseError(filename.clone(), url.to_string(), Box::new(err)) })?; Ok(metadata) } @@ -364,7 +445,7 @@ impl RegistryClient { &self, url: &Url, ) -> Result, Error> { - if self.index_urls.no_index() { + if self.index_locations.no_index() { return Err(Error::NoIndex(url.to_string())); } @@ -413,7 +494,7 @@ pub async fn read_metadata_async( .map_err(|err| Error::Zip(filename.clone(), err))?; let metadata = Metadata21::parse(&contents) - .map_err(|err| Error::MetadataParseError(filename.clone(), debug_source, err))?; + .map_err(|err| Error::MetadataParseError(filename.clone(), debug_source, Box::new(err)))?; Ok(metadata) } diff --git a/crates/puffin-dev/src/build.rs b/crates/puffin-dev/src/build.rs index 06bad8010..8415f2c1d 100644 --- a/crates/puffin-dev/src/build.rs +++ b/crates/puffin-dev/src/build.rs @@ -5,7 +5,7 @@ use anyhow::{Context, Result}; use clap::Parser; use fs_err as fs; -use distribution_types::IndexUrls; +use distribution_types::IndexLocations; use platform_host::Platform; use puffin_build::{SourceBuild, SourceBuildContext}; use puffin_cache::{Cache, CacheArgs}; @@ -54,7 +54,7 @@ pub(crate) async fn build(args: BuildArgs) -> Result { let platform = Platform::current()?; let venv = Virtualenv::from_env(platform, &cache)?; let client = RegistryClientBuilder::new(cache.clone()).build(); - let index_urls = IndexUrls::default(); + let index_urls = IndexLocations::default(); let setup_py = SetupPyStrategy::default(); let build_dispatch = BuildDispatch::new( diff --git a/crates/puffin-dev/src/install_many.rs b/crates/puffin-dev/src/install_many.rs index eacf65335..6a62f0381 100644 --- a/crates/puffin-dev/src/install_many.rs +++ b/crates/puffin-dev/src/install_many.rs @@ -11,7 +11,7 @@ use rustc_hash::FxHashMap; use tracing::info; use distribution_types::{ - CachedDist, Dist, DistributionMetadata, IndexUrls, Name, Resolution, VersionOrUrl, + CachedDist, Dist, DistributionMetadata, IndexLocations, Name, Resolution, VersionOrUrl, }; use install_wheel_rs::linker::LinkMode; use pep508_rs::Requirement; @@ -59,7 +59,7 @@ pub(crate) async fn install_many(args: InstallManyArgs) -> Result<()> { let platform = Platform::current()?; let venv = Virtualenv::from_env(platform, &cache)?; let client = RegistryClientBuilder::new(cache.clone()).build(); - let index_urls = IndexUrls::default(); + let index_locations = IndexLocations::default(); let setup_py = SetupPyStrategy::default(); let tags = venv.interpreter().tags()?; @@ -67,7 +67,7 @@ pub(crate) async fn install_many(args: InstallManyArgs) -> Result<()> { &client, &cache, venv.interpreter(), - &index_urls, + &index_locations, venv.python_executable(), setup_py, args.no_build, @@ -81,7 +81,7 @@ pub(crate) async fn install_many(args: InstallManyArgs) -> Result<()> { tags, &client, &venv, - &index_urls, + &index_locations, ) .await { @@ -101,7 +101,7 @@ async fn install_chunk( tags: &Tags, client: &RegistryClient, venv: &Virtualenv, - index_urls: &IndexUrls, + index_locations: &IndexLocations, ) -> Result<()> { let resolution: Vec<_> = DistFinder::new(tags, client, venv.interpreter()) .resolve_stream(requirements) @@ -136,7 +136,7 @@ async fn install_chunk( .into_distributions() .collect::>(); - let mut registry_index = RegistryWheelIndex::new(build_dispatch.cache(), tags, index_urls); + let mut registry_index = RegistryWheelIndex::new(build_dispatch.cache(), tags, index_locations); let (cached, uncached): (Vec<_>, Vec<_>) = dists.into_iter().partition_map(|dist| { // We always want the wheel for the latest version not whatever matching is in cache let VersionOrUrl::Version(version) = dist.version_or_url() else { diff --git a/crates/puffin-dev/src/resolve_cli.rs b/crates/puffin-dev/src/resolve_cli.rs index dfffc30f0..02e769dd3 100644 --- a/crates/puffin-dev/src/resolve_cli.rs +++ b/crates/puffin-dev/src/resolve_cli.rs @@ -9,7 +9,7 @@ use fs_err::File; use itertools::Itertools; use petgraph::dot::{Config as DotConfig, Dot}; -use distribution_types::{IndexUrls, Resolution}; +use distribution_types::{FlatIndexLocation, IndexLocations, IndexUrl, Resolution}; use pep508_rs::Requirement; use platform_host::Platform; use puffin_cache::{Cache, CacheArgs}; @@ -42,6 +42,12 @@ pub(crate) struct ResolveCliArgs { cache_args: CacheArgs, #[arg(long)] exclude_newer: Option>, + #[clap(long, short, default_value = IndexUrl::Pypi.as_str(), env = "PUFFIN_INDEX_URL")] + index_url: IndexUrl, + #[clap(long)] + extra_index_url: Vec, + #[clap(long)] + find_links: Vec, } pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> { @@ -49,17 +55,19 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> { let platform = Platform::current()?; let venv = Virtualenv::from_env(platform, &cache)?; - let client = RegistryClientBuilder::new(cache.clone()).build(); - let index_urls = IndexUrls::default(); - let setup_py = SetupPyStrategy::default(); + let index_locations = + IndexLocations::from_args(args.index_url, args.extra_index_url, args.find_links, false); + let client = RegistryClientBuilder::new(cache.clone()) + .index_locations(index_locations.clone()) + .build(); let build_dispatch = BuildDispatch::new( &client, &cache, venv.interpreter(), - &index_urls, + &index_locations, venv.python_executable(), - setup_py, + SetupPyStrategy::default(), args.no_build, ); @@ -73,7 +81,7 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> { tags, &client, &build_dispatch, - ); + )?; let resolution_graph = resolver.resolve().await.with_context(|| { format!( "No solution found when resolving: {}", diff --git a/crates/puffin-dev/src/resolve_many.rs b/crates/puffin-dev/src/resolve_many.rs index 8112f13df..a51a0b0bb 100644 --- a/crates/puffin-dev/src/resolve_many.rs +++ b/crates/puffin-dev/src/resolve_many.rs @@ -11,7 +11,7 @@ use tokio::time::Instant; use tracing::{info, info_span, Span}; use tracing_indicatif::span_ext::IndicatifSpanExt; -use distribution_types::IndexUrls; +use distribution_types::IndexLocations; use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers}; use pep508_rs::{Requirement, VersionOrUrl}; use platform_host::Platform; @@ -73,14 +73,14 @@ pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> { let platform = Platform::current()?; let venv = Virtualenv::from_env(platform, &cache)?; let client = RegistryClientBuilder::new(cache.clone()).build(); - let index_urls = IndexUrls::default(); + let index_locations = IndexLocations::default(); let setup_py = SetupPyStrategy::default(); let build_dispatch = BuildDispatch::new( &client, &cache, venv.interpreter(), - &index_urls, + &index_locations, venv.python_executable(), setup_py, args.no_build, diff --git a/crates/puffin-dispatch/src/lib.rs b/crates/puffin-dispatch/src/lib.rs index b1bee5aac..583e39766 100644 --- a/crates/puffin-dispatch/src/lib.rs +++ b/crates/puffin-dispatch/src/lib.rs @@ -9,7 +9,7 @@ use anyhow::{bail, Context, Result}; use itertools::Itertools; use tracing::{debug, instrument}; -use distribution_types::{CachedDist, DistributionId, IndexUrls, Name, Resolution}; +use distribution_types::{CachedDist, DistributionId, IndexLocations, Name, Resolution}; use pep508_rs::Requirement; use puffin_build::{SourceBuild, SourceBuildContext}; use puffin_cache::Cache; @@ -25,7 +25,7 @@ pub struct BuildDispatch<'a> { client: &'a RegistryClient, cache: &'a Cache, interpreter: &'a Interpreter, - index_urls: &'a IndexUrls, + index_locations: &'a IndexLocations, base_python: PathBuf, setup_py: SetupPyStrategy, no_build: bool, @@ -39,7 +39,7 @@ impl<'a> BuildDispatch<'a> { client: &'a RegistryClient, cache: &'a Cache, interpreter: &'a Interpreter, - index_urls: &'a IndexUrls, + index_locations: &'a IndexLocations, base_python: PathBuf, setup_py: SetupPyStrategy, no_build: bool, @@ -48,7 +48,7 @@ impl<'a> BuildDispatch<'a> { client, cache, interpreter, - index_urls, + index_locations, base_python, setup_py, no_build, @@ -99,7 +99,7 @@ impl<'a> BuildContext for BuildDispatch<'a> { tags, self.client, self, - ); + )?; let graph = resolver.resolve().await.with_context(|| { format!( "No solution found when resolving: {}", @@ -149,7 +149,7 @@ impl<'a> BuildContext for BuildDispatch<'a> { Vec::new(), site_packages, &Reinstall::None, - self.index_urls, + self.index_locations, self.cache(), venv, tags, diff --git a/crates/puffin-distribution/src/distribution_database.rs b/crates/puffin-distribution/src/distribution_database.rs index c9ef5c5f6..696a27a38 100644 --- a/crates/puffin-distribution/src/distribution_database.rs +++ b/crates/puffin-distribution/src/distribution_database.rs @@ -12,7 +12,9 @@ use tracing::instrument; use url::Url; use distribution_filename::{WheelFilename, WheelFilenameError}; -use distribution_types::{BuiltDist, DirectGitUrl, Dist, LocalEditable, Name, SourceDist}; +use distribution_types::{ + BuiltDist, DirectGitUrl, Dist, FileLocation, LocalEditable, Name, SourceDist, +}; use platform_tags::Tags; use puffin_cache::{Cache, CacheBucket, WheelCache}; use puffin_client::RegistryClient; @@ -108,6 +110,24 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context> ) -> Result { match &dist { Dist::Built(BuiltDist::Registry(wheel)) => { + let url = match &wheel.file.url { + FileLocation::Url(url) => url, + FileLocation::Path(path, url) => { + let cache_entry = self.cache.entry( + CacheBucket::Wheels, + WheelCache::Url(url).remote_wheel_dir(wheel.name().as_ref()), + wheel.filename.stem(), + ); + + return Ok(LocalWheel::Disk(DiskWheel { + dist: dist.clone(), + path: path.clone(), + target: cache_entry.into_path_buf(), + filename: wheel.filename.clone(), + })); + } + }; + // Download and unzip on the same tokio task. // // In all wheels we've seen so far, unzipping while downloading is @@ -123,7 +143,7 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context> // for downloading and unzipping (with a buffer in between) and switch // to rayon if this buffer grows large by the time the file is fully // downloaded. - let reader = self.client.stream_external(&wheel.file.url).await?; + let reader = self.client.stream_external(url).await?; // Download and unzip the wheel to a temporary directory. let temp_dir = tempfile::tempdir_in(self.cache.root())?; diff --git a/crates/puffin-distribution/src/index/registry_wheel_index.rs b/crates/puffin-distribution/src/index/registry_wheel_index.rs index 1b34069c8..93068ee55 100644 --- a/crates/puffin-distribution/src/index/registry_wheel_index.rs +++ b/crates/puffin-distribution/src/index/registry_wheel_index.rs @@ -4,7 +4,7 @@ use std::path::Path; use rustc_hash::FxHashMap; -use distribution_types::{CachedRegistryDist, CachedWheel, IndexUrls}; +use distribution_types::{CachedRegistryDist, CachedWheel, IndexLocations}; use pep440_rs::Version; use platform_tags::Tags; use puffin_cache::{Cache, CacheBucket, WheelCache}; @@ -16,17 +16,17 @@ use puffin_normalize::PackageName; pub struct RegistryWheelIndex<'a> { cache: &'a Cache, tags: &'a Tags, - index_urls: &'a IndexUrls, + index_locations: &'a IndexLocations, index: FxHashMap>, } impl<'a> RegistryWheelIndex<'a> { /// Initialize an index of cached distributions from a directory. - pub fn new(cache: &'a Cache, tags: &'a Tags, index_urls: &'a IndexUrls) -> Self { + pub fn new(cache: &'a Cache, tags: &'a Tags, index_locations: &'a IndexLocations) -> Self { Self { cache, tags, - index_urls, + index_locations, index: FxHashMap::default(), } } @@ -56,9 +56,12 @@ impl<'a> RegistryWheelIndex<'a> { fn get_impl(&mut self, name: &PackageName) -> &BTreeMap { let versions = match self.index.entry(name.clone()) { Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(entry) => { - entry.insert(Self::index(name, self.cache, self.tags, self.index_urls)) - } + Entry::Vacant(entry) => entry.insert(Self::index( + name, + self.cache, + self.tags, + self.index_locations, + )), }; versions } @@ -68,11 +71,11 @@ impl<'a> RegistryWheelIndex<'a> { package: &PackageName, cache: &Cache, tags: &Tags, - index_urls: &IndexUrls, + index_locations: &IndexLocations, ) -> BTreeMap { let mut versions = BTreeMap::new(); - for index_url in index_urls { + for index_url in index_locations.indexes() { // Index all the wheels that were downloaded directly from the registry. let wheel_dir = cache.shard( CacheBucket::Wheels, diff --git a/crates/puffin-distribution/src/source/mod.rs b/crates/puffin-distribution/src/source/mod.rs index ee9a2bff1..7363488db 100644 --- a/crates/puffin-distribution/src/source/mod.rs +++ b/crates/puffin-distribution/src/source/mod.rs @@ -16,8 +16,8 @@ use zip::ZipArchive; use distribution_filename::WheelFilename; use distribution_types::{ - DirectArchiveUrl, DirectGitUrl, Dist, GitSourceDist, Identifier, LocalEditable, Name, - PathSourceDist, RemoteSource, SourceDist, + DirectArchiveUrl, DirectGitUrl, Dist, FileLocation, GitSourceDist, Identifier, LocalEditable, + Name, PathSourceDist, RemoteSource, SourceDist, }; use install_wheel_rs::read_dist_info; use platform_tags::Tags; @@ -98,19 +98,32 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> { .await? } SourceDist::Registry(registry_source_dist) => { + let url = match ®istry_source_dist.file.url { + FileLocation::Url(url) => url, + FileLocation::Path(path, url) => { + let path_source_dist = PathSourceDist { + name: registry_source_dist.filename.name.clone(), + url: url.clone(), + path: path.clone(), + editable: false, + }; + return self.path(source_dist, &path_source_dist).await; + } + }; + // For registry source distributions, shard by package, then by SHA. // Ex) `pypi/requests/a673187abc19fe6c` let cache_shard = self.build_context.cache().shard( CacheBucket::BuiltWheels, WheelCache::Index(®istry_source_dist.index) - .remote_wheel_dir(registry_source_dist.name.as_ref()) + .remote_wheel_dir(registry_source_dist.filename.name.as_ref()) .join(®istry_source_dist.file.distribution_id().as_str()[..16]), ); self.url( source_dist, ®istry_source_dist.file.filename, - ®istry_source_dist.file.url, + url, &cache_shard, None, ) @@ -154,19 +167,32 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> { .await? } SourceDist::Registry(registry_source_dist) => { + let url = match ®istry_source_dist.file.url { + FileLocation::Url(url) => url, + FileLocation::Path(path, url) => { + let path_source_dist = PathSourceDist { + name: registry_source_dist.filename.name.clone(), + url: url.clone(), + path: path.clone(), + editable: false, + }; + return self.path_metadata(source_dist, &path_source_dist).await; + } + }; + // For registry source distributions, shard by package, then by SHA. // Ex) `pypi/requests/a673187abc19fe6c` let cache_shard = self.build_context.cache().shard( CacheBucket::BuiltWheels, WheelCache::Index(®istry_source_dist.index) - .remote_wheel_dir(registry_source_dist.name.as_ref()) + .remote_wheel_dir(registry_source_dist.filename.name.as_ref()) .join(®istry_source_dist.file.distribution_id().as_str()[..16]), ); self.url_metadata( source_dist, ®istry_source_dist.file.filename, - ®istry_source_dist.file.url, + url, &cache_shard, None, ) diff --git a/crates/puffin-installer/src/downloader.rs b/crates/puffin-installer/src/downloader.rs index 81e592909..f46f0a183 100644 --- a/crates/puffin-installer/src/downloader.rs +++ b/crates/puffin-installer/src/downloader.rs @@ -153,7 +153,7 @@ impl<'a, Context: BuildContext + Send + Sync> Downloader<'a, Context> { } /// Download, build, and unzip a single wheel. - #[instrument(skip_all, fields(name = % dist, size = ? dist.size(), url = dist.file().map(|file| file.url.as_str()).unwrap_or_default()))] + #[instrument(skip_all, fields(name = % dist, size = ? dist.size(), url = dist.file().map(|file| file.url.to_string()).unwrap_or_default()))] pub async fn get_wheel( &self, dist: Dist, diff --git a/crates/puffin-installer/src/plan.rs b/crates/puffin-installer/src/plan.rs index 40e493a15..e28b26049 100644 --- a/crates/puffin-installer/src/plan.rs +++ b/crates/puffin-installer/src/plan.rs @@ -5,7 +5,7 @@ use rustc_hash::FxHashSet; use tracing::{debug, warn}; use distribution_types::{ - git_reference, BuiltDist, CachedDirectUrlDist, CachedDist, Dist, IndexUrls, InstalledDist, + git_reference, BuiltDist, CachedDirectUrlDist, CachedDist, Dist, IndexLocations, InstalledDist, Name, SourceDist, }; use pep508_rs::{Requirement, VersionOrUrl}; @@ -45,13 +45,13 @@ impl InstallPlan { editable_requirements: Vec, mut site_packages: SitePackages, reinstall: &Reinstall, - index_urls: &IndexUrls, + index_locations: &IndexLocations, cache: &Cache, venv: &Virtualenv, tags: &Tags, ) -> Result { // Index all the already-downloaded wheels in the cache. - let mut registry_index = RegistryWheelIndex::new(cache, tags, index_urls); + let mut registry_index = RegistryWheelIndex::new(cache, tags, index_locations); let mut local = vec![]; let mut remote = vec![]; diff --git a/crates/puffin-resolver/src/candidate_selector.rs b/crates/puffin-resolver/src/candidate_selector.rs index 017d5aef3..8292e7c0d 100644 --- a/crates/puffin-resolver/src/candidate_selector.rs +++ b/crates/puffin-resolver/src/candidate_selector.rs @@ -2,6 +2,7 @@ use pubgrub::range::Range; use rustc_hash::FxHashMap; use distribution_types::{Dist, DistributionMetadata, Name}; +use distribution_types::{DistRequiresPython, ResolvableDist}; use pep440_rs::VersionSpecifiers; use pep508_rs::{Requirement, VersionOrUrl}; use puffin_normalize::PackageName; @@ -10,7 +11,7 @@ use crate::prerelease_mode::PreReleaseStrategy; use crate::pubgrub::PubGrubVersion; use crate::python_requirement::PythonRequirement; use crate::resolution_mode::ResolutionStrategy; -use crate::version_map::{DistRequiresPython, ResolvableFile, VersionMap}; +use crate::version_map::VersionMap; use crate::{Manifest, ResolutionOptions}; #[derive(Debug, Clone)] @@ -160,7 +161,7 @@ impl CandidateSelector { /// Select the first-matching [`Candidate`] from a set of candidate versions and files, /// preferring wheels over source distributions. fn select_candidate<'a>( - versions: impl Iterator)>, + versions: impl Iterator)>, package_name: &'a PackageName, range: &Range, allow_prerelease: AllowPreRelease, @@ -168,7 +169,7 @@ impl CandidateSelector { #[derive(Debug)] enum PreReleaseCandidate<'a> { NotNecessary, - IfNecessary(&'a PubGrubVersion, ResolvableFile<'a>), + IfNecessary(&'a PubGrubVersion, ResolvableDist<'a>), } let mut prerelease = None; @@ -222,15 +223,15 @@ pub(crate) struct Candidate<'a> { /// The version of the package. version: &'a PubGrubVersion, /// The file to use for resolving and installing the package. - file: ResolvableFile<'a>, + dist: ResolvableDist<'a>, } impl<'a> Candidate<'a> { - fn new(name: &'a PackageName, version: &'a PubGrubVersion, file: ResolvableFile<'a>) -> Self { + fn new(name: &'a PackageName, version: &'a PubGrubVersion, dist: ResolvableDist<'a>) -> Self { Self { name, version, - file, + dist, } } @@ -246,12 +247,12 @@ impl<'a> Candidate<'a> { /// Return the [`DistFile`] to use when resolving the package. pub(crate) fn resolve(&self) -> &DistRequiresPython { - self.file.resolve() + self.dist.resolve() } /// Return the [`DistFile`] to use when installing the package. pub(crate) fn install(&self) -> &DistRequiresPython { - self.file.install() + self.dist.install() } /// If the candidate doesn't match the given requirement, return the version specifiers. diff --git a/crates/puffin-resolver/src/finder.rs b/crates/puffin-resolver/src/finder.rs index 6e73ea384..95d7c739e 100644 --- a/crates/puffin-resolver/src/finder.rs +++ b/crates/puffin-resolver/src/finder.rs @@ -3,6 +3,7 @@ //! This is similar to running `pip install` with the `--no-deps` flag. use anyhow::Result; +use distribution_filename::DistFilename; use futures::{stream, Stream, StreamExt, TryStreamExt}; use rustc_hash::FxHashMap; @@ -56,15 +57,10 @@ impl<'a> DistFinder<'a> { let (index, metadata) = self.client.simple(&requirement.name).await?; // Pick a version that satisfies the requirement. - let Some(ParsedFile { - name, - version, - file, - }) = self.select(requirement, metadata) - else { + let Some(ParsedFile { filename, file }) = self.select(requirement, metadata) else { return Err(ResolveError::NotFound(requirement.clone())); }; - let distribution = Dist::from_registry(name, version, file, index); + let distribution = Dist::from_registry(filename, file, index); if let Some(reporter) = self.reporter.as_ref() { reporter.on_progress(&distribution); @@ -152,8 +148,7 @@ impl<'a> DistFinder<'a> { { best_wheel = Some(( ParsedFile { - name: wheel.name, - version: wheel.version, + filename: DistFilename::WheelFilename(wheel), file, }, priority, @@ -181,8 +176,7 @@ impl<'a> DistFinder<'a> { best_version = Some(sdist.version.clone()); best_sdist = Some(ParsedFile { - name: sdist.name, - version: sdist.version, + filename: DistFilename::SourceDistFilename(sdist), file, }); } @@ -195,10 +189,8 @@ impl<'a> DistFinder<'a> { #[derive(Debug)] struct ParsedFile { - /// The [`PackageName`] extracted from the [`File`]. - name: PackageName, - /// The version extracted from the [`File`]. - version: Version, + /// The wheel or source dist filename extracted from the [`File`]. + filename: DistFilename, /// The underlying [`File`]. file: File, } diff --git a/crates/puffin-resolver/src/resolution.rs b/crates/puffin-resolver/src/resolution.rs index ee0ebb0cf..49d37a348 100644 --- a/crates/puffin-resolver/src/resolution.rs +++ b/crates/puffin-resolver/src/resolution.rs @@ -253,12 +253,12 @@ impl std::fmt::Display for DisplayResolutionGraph<'_> { nodes.sort_unstable_by_key(|(_, package)| package.name()); // Print out the dependency graph. - for (index, package) in nodes { + for (index, dist) in nodes { // Display the node itself. - if let Some((editable, _)) = self.resolution.editables.get(package.name()) { + if let Some((editable, _)) = self.resolution.editables.get(dist.name()) { write!(f, "-e {}", editable.verbatim())?; } else { - write!(f, "{}", package.verbatim())?; + write!(f, "{}", dist.verbatim())?; } // Display the distribution hashes, if any. @@ -266,7 +266,7 @@ impl std::fmt::Display for DisplayResolutionGraph<'_> { if let Some(hashes) = self .resolution .hashes - .get(package.name()) + .get(dist.name()) .filter(|hashes| !hashes.is_empty()) { for hash in hashes { diff --git a/crates/puffin-resolver/src/resolver/mod.rs b/crates/puffin-resolver/src/resolver/mod.rs index d60756bbc..0cce39d39 100644 --- a/crates/puffin-resolver/src/resolver/mod.rs +++ b/crates/puffin-resolver/src/resolver/mod.rs @@ -75,6 +75,8 @@ pub struct Resolver<'a, Provider: ResolverProvider> { impl<'a, Context: BuildContext + Send + Sync> Resolver<'a, DefaultResolverProvider<'a, Context>> { /// Initialize a new resolver using the default backend doing real requests. + /// + /// Reads the flat index entries. pub fn new( manifest: Manifest, options: ResolutionOptions, @@ -83,7 +85,7 @@ impl<'a, Context: BuildContext + Send + Sync> Resolver<'a, DefaultResolverProvid tags: &'a Tags, client: &'a RegistryClient, build_context: &'a Context, - ) -> Self { + ) -> Result { let provider = DefaultResolverProvider::new( client, DistributionDatabase::new(build_context.cache(), tags, client, build_context), @@ -95,14 +97,14 @@ impl<'a, Context: BuildContext + Send + Sync> Resolver<'a, DefaultResolverProvid .iter() .chain(manifest.constraints.iter()) .collect(), - ); - Self::new_custom_io( + )?; + Ok(Self::new_custom_io( manifest, options, markers, PythonRequirement::new(interpreter, markers), provider, - ) + )) } } @@ -377,14 +379,10 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> { } PubGrubPackage::Package(package_name, _extra, Some(url)) => { // Emit a request to fetch the metadata for this distribution. - let distribution = Dist::from_url(package_name.clone(), url.clone())?; - if self - .index - .distributions - .register_owned(distribution.package_id()) - { - priorities.add(distribution.name().clone()); - request_sink.unbounded_send(Request::Dist(distribution))?; + let dist = Dist::from_url(package_name.clone(), url.clone())?; + if self.index.distributions.register_owned(dist.package_id()) { + priorities.add(dist.name().clone()); + request_sink.unbounded_send(Request::Dist(dist))?; } } } @@ -542,8 +540,8 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> { .distributions .register_owned(candidate.package_id()) { - let distribution = candidate.resolve().dist.clone(); - request_sink.unbounded_send(Request::Dist(distribution))?; + let dist = candidate.resolve().dist.clone(); + request_sink.unbounded_send(Request::Dist(dist))?; } Ok(Some(version)) @@ -690,13 +688,19 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> { trace!("Received package metadata for: {package_name}"); self.index.packages.done(package_name, version_map); } - Some(Response::Dist(Dist::Built(distribution), metadata, ..)) => { - trace!("Received built distribution metadata for: {distribution}"); - self.index - .distributions - .done(distribution.package_id(), metadata); + Some(Response::Dist { + dist: Dist::Built(dist), + metadata, + precise: _, + }) => { + trace!("Received built distribution metadata for: {dist}"); + self.index.distributions.done(dist.package_id(), metadata); } - Some(Response::Dist(Dist::Source(distribution), metadata, precise)) => { + Some(Response::Dist { + dist: Dist::Source(distribution), + metadata, + precise, + }) => { trace!("Received source distribution metadata for: {distribution}"); self.index .distributions @@ -753,7 +757,11 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> { ResolveError::FetchAndBuild(Box::new(source_dist), err) } })?; - Ok(Some(Response::Dist(dist, metadata, precise))) + Ok(Some(Response::Dist { + dist, + metadata, + precise, + })) } // Pre-fetch the package and distribution metadata. @@ -804,7 +812,11 @@ impl<'a, Provider: ResolverProvider> Resolver<'a, Provider> { } })?; - Ok(Some(Response::Dist(dist, metadata, precise))) + Ok(Some(Response::Dist { + dist, + metadata, + precise, + })) } else { Ok(None) } @@ -852,7 +864,11 @@ enum Response { /// The returned metadata for a package hosted on a registry. Package(PackageName, VersionMap), /// The returned metadata for a distribution. - Dist(Dist, Metadata21, Option), + Dist { + dist: Dist, + metadata: Metadata21, + precise: Option, + }, } /// An enum used by [`DependencyProvider`] that holds information about package dependencies. diff --git a/crates/puffin-resolver/src/resolver/provider.rs b/crates/puffin-resolver/src/resolver/provider.rs index e5359f660..cb3579141 100644 --- a/crates/puffin-resolver/src/resolver/provider.rs +++ b/crates/puffin-resolver/src/resolver/provider.rs @@ -3,11 +3,13 @@ use std::future::Future; use anyhow::Result; use chrono::{DateTime, Utc}; use futures::TryFutureExt; +use rustc_hash::FxHashMap; use url::Url; +use crate::pubgrub::PubGrubVersion; use distribution_types::Dist; use platform_tags::Tags; -use puffin_client::RegistryClient; +use puffin_client::{FlatIndex, RegistryClient}; use puffin_distribution::{DistributionDatabase, DistributionDatabaseError}; use puffin_normalize::PackageName; use puffin_traits::BuildContext; @@ -46,6 +48,8 @@ pub trait ResolverProvider: Send + Sync { /// [`RegistryClient`] and [`DistributionDatabase`]. pub struct DefaultResolverProvider<'a, Context: BuildContext + Send + Sync> { client: &'a RegistryClient, + /// These are the entries from `--find-links` that act as overrides for index responses. + flat_index: FxHashMap>, fetcher: DistributionDatabase<'a, Context>, tags: &'a Tags, python_requirement: PythonRequirement<'a>, @@ -54,6 +58,7 @@ pub struct DefaultResolverProvider<'a, Context: BuildContext + Send + Sync> { } impl<'a, Context: BuildContext + Send + Sync> DefaultResolverProvider<'a, Context> { + /// Reads the flat index entries and builds the provider. pub fn new( client: &'a RegistryClient, fetcher: DistributionDatabase<'a, Context>, @@ -61,15 +66,19 @@ impl<'a, Context: BuildContext + Send + Sync> DefaultResolverProvider<'a, Contex python_requirement: PythonRequirement<'a>, exclude_newer: Option>, allowed_yanks: AllowedYanks, - ) -> Self { - Self { + ) -> Result { + let flat_index_dists = client.flat_index()?; + let flat_index = FlatIndex::from_dists(flat_index_dists, tags); + + Ok(Self { client, + flat_index, fetcher, tags, python_requirement, exclude_newer, allowed_yanks, - } + }) } } @@ -80,6 +89,7 @@ impl<'a, Context: BuildContext + Send + Sync> ResolverProvider &'io self, package_name: &'io PackageName, ) -> impl Future + Send + 'io { + let flat_index_override = self.flat_index.get(package_name).cloned(); self.client .simple(package_name) .map_ok(move |(index, metadata)| { @@ -91,6 +101,7 @@ impl<'a, Context: BuildContext + Send + Sync> ResolverProvider &self.python_requirement, &self.allowed_yanks, self.exclude_newer.as_ref(), + flat_index_override, ) }) } diff --git a/crates/puffin-resolver/src/version_map.rs b/crates/puffin-resolver/src/version_map.rs index e3e5903e4..5bbbe2abc 100644 --- a/crates/puffin-resolver/src/version_map.rs +++ b/crates/puffin-resolver/src/version_map.rs @@ -5,10 +5,9 @@ use chrono::{DateTime, Utc}; use tracing::{instrument, warn}; use distribution_filename::DistFilename; -use distribution_types::{Dist, IndexUrl}; -use pep440_rs::VersionSpecifiers; -use platform_tags::{TagPriority, Tags}; -use puffin_client::SimpleMetadata; +use distribution_types::{Dist, IndexUrl, PrioritizedDistribution, ResolvableDist}; +use platform_tags::Tags; +use puffin_client::{FlatIndex, SimpleMetadata}; use puffin_normalize::PackageName; use puffin_warnings::warn_user_once; use pypi_types::{Hashes, Yanked}; @@ -18,12 +17,12 @@ use crate::python_requirement::PythonRequirement; use crate::yanks::AllowedYanks; /// A map from versions to distributions. -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct VersionMap(BTreeMap); impl VersionMap { /// Initialize a [`VersionMap`] from the given metadata. - #[instrument(skip_all, fields(package_name = % package_name))] + #[instrument(skip_all, fields(package_name))] #[allow(clippy::too_many_arguments)] pub(crate) fn from_metadata( metadata: SimpleMetadata, @@ -33,9 +32,11 @@ impl VersionMap { python_requirement: &PythonRequirement, allowed_yanks: &AllowedYanks, exclude_newer: Option<&DateTime>, + flat_index: Option>, ) -> Self { + // If we have packages of the same name from find links, gives them priority, otherwise start empty let mut version_map: BTreeMap = - BTreeMap::default(); + flat_index.map(|overrides| overrides.0).unwrap_or_default(); // Collect compatible distributions. for (version, files) in metadata { @@ -82,22 +83,24 @@ impl VersionMap { }) }); let dist = Dist::from_registry( - filename.name.clone(), - filename.version.clone(), + DistFilename::WheelFilename(filename), file, index.clone(), ); match version_map.entry(version.clone().into()) { Entry::Occupied(mut entry) => { - entry - .get_mut() - .insert_built(dist, requires_python, hash, priority); + entry.get_mut().insert_built( + dist, + requires_python, + Some(hash), + priority, + ); } Entry::Vacant(entry) => { entry.insert(PrioritizedDistribution::from_built( dist, requires_python, - hash, + Some(hash), priority, )); } @@ -105,20 +108,21 @@ impl VersionMap { } DistFilename::SourceDistFilename(filename) => { let dist = Dist::from_registry( - filename.name.clone(), - filename.version.clone(), + DistFilename::SourceDistFilename(filename), file, index.clone(), ); match version_map.entry(version.clone().into()) { Entry::Occupied(mut entry) => { - entry.get_mut().insert_source(dist, requires_python, hash); + entry + .get_mut() + .insert_source(dist, requires_python, Some(hash)); } Entry::Vacant(entry) => { entry.insert(PrioritizedDistribution::from_source( dist, requires_python, - hash, + Some(hash), )); } } @@ -131,200 +135,24 @@ impl VersionMap { } /// Return the [`DistFile`] for the given version, if any. - pub(crate) fn get(&self, version: &PubGrubVersion) -> Option { + pub(crate) fn get(&self, version: &PubGrubVersion) -> Option { self.0.get(version).and_then(PrioritizedDistribution::get) } /// Return an iterator over the versions and distributions. pub(crate) fn iter( &self, - ) -> impl DoubleEndedIterator { + ) -> impl DoubleEndedIterator { self.0 .iter() - .filter_map(|(version, file)| Some((version, file.get()?))) + .filter_map(|(version, dist)| Some((version, dist.get()?))) } /// Return the [`Hashes`] for the given version, if any. pub(crate) fn hashes(&self, version: &PubGrubVersion) -> Vec { self.0 .get(version) - .map(|file| file.hashes.clone()) + .map(|file| file.hashes().to_vec()) .unwrap_or_default() } } - -/// Attach its requires-python to a [`Dist`], since downstream needs this information to filter -/// [`PrioritizedDistribution`]. -#[derive(Debug)] -pub(crate) struct DistRequiresPython { - pub(crate) dist: Dist, - pub(crate) requires_python: Option, -} - -#[derive(Debug)] -struct PrioritizedDistribution { - /// An arbitrary source distribution for the package version. - source: Option, - /// The highest-priority, platform-compatible wheel for the package version. - compatible_wheel: Option<(DistRequiresPython, TagPriority)>, - /// An arbitrary, platform-incompatible wheel for the package version. - incompatible_wheel: Option, - /// The hashes for each distribution. - hashes: Vec, -} - -impl PrioritizedDistribution { - /// Create a new [`PrioritizedDistribution`] from the given wheel distribution. - fn from_built( - dist: Dist, - requires_python: Option, - hash: Hashes, - priority: Option, - ) -> Self { - if let Some(priority) = priority { - Self { - source: None, - compatible_wheel: Some(( - DistRequiresPython { - dist, - - requires_python, - }, - priority, - )), - incompatible_wheel: None, - hashes: vec![hash], - } - } else { - Self { - source: None, - compatible_wheel: None, - incompatible_wheel: Some(DistRequiresPython { - dist, - requires_python, - }), - hashes: vec![hash], - } - } - } - - /// Create a new [`PrioritizedDistribution`] from the given source distribution. - fn from_source(dist: Dist, requires_python: Option, hash: Hashes) -> Self { - Self { - source: Some(DistRequiresPython { - dist, - requires_python, - }), - compatible_wheel: None, - incompatible_wheel: None, - hashes: vec![hash], - } - } - - /// Insert the given built distribution into the [`PrioritizedDistribution`]. - fn insert_built( - &mut self, - dist: Dist, - requires_python: Option, - hash: Hashes, - priority: Option, - ) { - // Prefer the highest-priority, platform-compatible wheel. - if let Some(priority) = priority { - if let Some((.., existing_priority)) = &self.compatible_wheel { - if priority > *existing_priority { - self.compatible_wheel = Some(( - DistRequiresPython { - dist, - requires_python, - }, - priority, - )); - } - } else { - self.compatible_wheel = Some(( - DistRequiresPython { - dist, - requires_python, - }, - priority, - )); - } - } else if self.incompatible_wheel.is_none() { - self.incompatible_wheel = Some(DistRequiresPython { - dist, - requires_python, - }); - } - self.hashes.push(hash); - } - - /// Insert the given source distribution into the [`PrioritizedDistribution`]. - fn insert_source( - &mut self, - dist: Dist, - requires_python: Option, - hash: Hashes, - ) { - if self.source.is_none() { - self.source = Some(DistRequiresPython { - dist, - requires_python, - }); - } - self.hashes.push(hash); - } - - /// Return the highest-priority distribution for the package version, if any. - fn get(&self) -> Option { - match ( - &self.compatible_wheel, - &self.source, - &self.incompatible_wheel, - ) { - // Prefer the highest-priority, platform-compatible wheel. - (Some((wheel, _)), _, _) => Some(ResolvableFile::CompatibleWheel(wheel)), - // If we have a compatible source distribution and an incompatible wheel, return the - // wheel. We assume that all distributions have the same metadata for a given package - // version. If a compatible source distribution exists, we assume we can build it, but - // using the wheel is faster. - (_, Some(source_dist), Some(wheel)) => { - Some(ResolvableFile::IncompatibleWheel(source_dist, wheel)) - } - // Otherwise, if we have a source distribution, return it. - (_, Some(source_dist), _) => Some(ResolvableFile::SourceDist(source_dist)), - _ => None, - } - } -} - -#[derive(Debug, Clone)] -pub(crate) enum ResolvableFile<'a> { - /// The distribution should be resolved and installed using a source distribution. - SourceDist(&'a DistRequiresPython), - /// The distribution should be resolved and installed using a wheel distribution. - CompatibleWheel(&'a DistRequiresPython), - /// The distribution should be resolved using an incompatible wheel distribution, but - /// installed using a source distribution. - IncompatibleWheel(&'a DistRequiresPython, &'a DistRequiresPython), -} - -impl<'a> ResolvableFile<'a> { - /// Return the [`DistFile`] to use during resolution. - pub(crate) fn resolve(&self) -> &DistRequiresPython { - match *self { - ResolvableFile::SourceDist(sdist) => sdist, - ResolvableFile::CompatibleWheel(wheel) => wheel, - ResolvableFile::IncompatibleWheel(_, wheel) => wheel, - } - } - - /// Return the [`DistFile`] to use during installation. - pub(crate) fn install(&self) -> &DistRequiresPython { - match *self { - ResolvableFile::SourceDist(sdist) => sdist, - ResolvableFile::CompatibleWheel(wheel) => wheel, - ResolvableFile::IncompatibleWheel(sdist, _) => sdist, - } - } -} diff --git a/crates/puffin-resolver/tests/resolver.rs b/crates/puffin-resolver/tests/resolver.rs index 0815ed226..141be6532 100644 --- a/crates/puffin-resolver/tests/resolver.rs +++ b/crates/puffin-resolver/tests/resolver.rs @@ -119,7 +119,7 @@ async fn resolve( tags, &client, &build_context, - ); + )?; Ok(resolver.resolve().await?) } diff --git a/scripts/wheels/maturin-1.4.0-py3-none-any.whl b/scripts/wheels/maturin-1.4.0-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..30cd9d288a15f4221ff50177b0688a91e98a16da GIT binary patch literal 5943 zcmZ{oWl&t%wtyRV2`-`W;O-I}g1fszLLnUGiqFg%^1G-`sg}NLJH&0Z`1}3CrDsx{FiH?1+7Oa%Vh=>U?6=w91K!I|R-D+( zkDWcrM8z+25@F^^uA+(e3RkJI#z)?n(&c$*=hIx4AUoZg^j#ioveJKZXgx-pqUAQD z3@ZyV*q-RtzfM=@2w~OT7aM83oV;Um%NX1fm_MkUD^$PQ=CYQoU-vg0kXBbTq6E$OniCc9mG zxMzHDL1kj(nF_eRvBPP7!2POnY{TdJ5WIyx?%8LcE_lGWj{kp5vc- zP)Al;TJiNzH!3V8_Pe!Km9-ko=*8MlHtZ#;{5W&=kL*IY1wBU)LT2Wt!;L^cz&KRg z&MbP-phQ`W;%kl6>#f8%K`rZ^YX-s1{jj;>ad=$a6 z^doC{t=JW4kxvkgUHb_Ylk-g66k+j9N{-m?<`Gk$ZgC_DPzlL-dV)eve~a%x}p>5YfK^ z9T@B=7U+z^9Zwb1C&JWA-x09&2p7@MzQfg}mF+iPF0%SY;d|Yu#*tJ=MQFw?IHXY+ zJ+i95g{)gKdYC?yz2LGFpX7J3-Nc@kU7QIsMLMmNA?uibaS?pollQ1@9i)Q82#?4y zIot3ksUVN$g@Z?LQLaX?)*w(SFvoB2l1`RhHM_!#-jQvfF2QuNsE4PJ5V_-U&6XQP zHqhPodDFrUG&{MEg$~ZKepjOjA5!$6 z9MdBjOUaY)1u~z`GbqXZUB^lNt8vZ%n*u)?!4;c2a_Pl`;SgiAn<6vwsfKD8&{uqU z&KdSrMDYE3s{i@5sG=-d%Njo3L1a^f*%}27XeOs3ZBHjl8=okd=1u$TAEis9WbvAC zlk(tE2yil}?kIMNkTC{gC*af|0XsorP`3PP@G8j&S+5L@%cku|^QSJsiM%tRJ%@)=@!IK3Qk^ z5tC5Bp@cxWL+l*onfv^$nW49MxEE*Z;UD#4XNKlf_bQs;QDciu7n{)zTtv~%V_EOz zS5Y&N-q1A@ev6r6$8U^5-%$m>8<({>uf^7~X|dVU?U8&ng`)2e`AW%mQR3RDgJhEm z!^5+JSeR%@T~kJMA5BW88AwFwu$W&Dpqb4ndQ#NYL-Il7?1cz5N;|28Wgar->7%$) zE3#TR_y*{LcoplOvmRV;i0w?}4Ra;*>^@R$>yX<#3T?hKiRQ-oVTDANVmbITZLp1p zc3aRYSKy_Q4~gO7NG~??r_x2zuqdUJ8Z3SJckE{;q=_d8gZhmmU};ToCAfue-=v(Q{H9g+&>|mc?*(`g>kVaqfIsR zSi>=?z_Z8`$jDlJEvJAmtVsw9wY{H-^D?zwT?`^qd#QbONUe3&LLh>eEW5OM`9@a+ zQ%>063vs#b%#@*aeXp*>TW3og;%6~h=0SY-(<92*zmKMtk4zhbCu-v`zLrllutb&i*p2+d zdmgcj8>-@*q5+GraS55>rBOP(7|~{!@AgjK!=Azllv%29+W?aWWHF|4C#cULN5*#4 zci2MUy6A;Ytui-kDU6$xYn0LE>+r^R3Z*^+`|Ve?#4QR%+zdEGH^oPH9udoMWW!4i z^hg8;9x(%$reFmTjSYO0lhMReH`e^B&%qp8alt|kN=!i?8)At|i&1jE>q>5h(5u=s zrHncJaX-;OJr;z3n4p_`5rkONt0tW}j*kUDrDiVWZe%+yluv9)Gn(&ja$#olm<8nL**OXHPOoSdG# zyOF!zCSDa_5e$`?L$kf^%M=6DciyQ`+`6ptvrHfh)lWpBa zvOR)UBuj7T&3V%Z!)GG8R&&r0N@NL#(i?c*5cTCcp<|Nt;cYepDm_r zRYJ=q+IV~D!KPn=p>ic-(5K-y(k`wl4O^nsr*sjq34K>E{cxNg1QWtdd}7)eIGU4N zt`)i3Q>sCm?sT2Y7qw7sz}4FEcs(_P3)U!B{HgfXA6i3PD+h1F#eZ5oVfta6N#k+YC>3fU6KbcsvBs?YC_a9% z?pcukp}3>@;avVw{h+;a_=+5yU*HY4(n7XEJzu{5W~Qq`04jHaa^C-ac7MQ29A~fkkH~*Q(xcOzB+XC_eDx`&D?cA>S94{A7fl zQoft0JN%3`cKj|+FT<-YjQ3WHB*C6zhMtglBJOr~)0ZRYeL0RKzGO)dv)d4o6S5_G zyPm6`<(qtezlCobUc^BV0f2RU0D#~x6Bo$Q#o5in#Mar_{(0z{A%ZwTcrTVrx*4z% zISdY~`QU&FU<~YBmSCPkc8?%21}1Y*{7_=g{K_hr(T1 zJ>CK!Et5h|tGKL^?FyO_4`F-y2vm>OXGCN_#rZz!K@hAOliu{GN&jS@>qDAX<``4l zKSCk>Z923PY&V-5tb9TCamfA}3T!W!%va2(!{wa;7Vg?MY`snvia(d1w)nn`KAg9| z)TN2q7F>Yx#~mq_fA1rSo=MGu;`b#xnbi#neGe9rK@n|-69;27f7bqv0Lr2Sq}@!_ ztvzG`4xe)5&b=^rm^xfHT9BuLm*f%g!laOlEfaCYTT8Ap0xt*R>&BKMJ_qp+V&&17 zq01=7xXXNPJLxc_gJ0AbBkeet#*7~?*AM1IhishLjExgLH^+q)f);|O%kwPC=633} z955NRShS)IiEvYgyBEGO74Ol)UuRzNB)i*NrdZh&!u^3`aqM87#4SNmA%q$=;9Hv) zJ+q96pvfhP=|(A}_E0;FZ=Zp%0Pu|r(SK1ydlS({@EY#tsXYv0BRYS@A2N{y$`u{j z`1cg=d)xudPia*Atbn(j!JMl@Et(1J-V(K`k31HT%X$l$c9)phs^UU#XRVb3ZjCu# zUPYSpA<|+UH6XAzGkJ?m4Jh=%lw&hS2Za+x;ma<1BP#!OB0B5@>3S;fNGL4c&UVj`(8+3u6i(rh*Tq6}k z%NZcSDXF@um(m`h_Zmp&FR(`9KIO57tMyC_$F4#MTi6ec<-Ik{0y9T=|$Trw^9^`%dFI17L7a)h!XiE*3kdR!o3BWcIn~u^XsftO z&g4HTStZj9%d0ar!We*MZrOHhr1dGK(U`*TterrB|41;Y5 z9J^HmogpWlC@A8T`^B9En!GzNNR|?{8AY$V*fpG`Lv;fXpus4*}&N8g`{Qn%Z=Kx?E4bozM5AvZVE;QWawjcCHo3=)0Vk9EW{FNKkR zm;=JXdYLV?ME)f|!+Qnco?4^Zco1~~G^h()YXKWXRfeRrP_d}4llWpttk^ikzQMXW zSVo^!B8V}`Jw<7<-_b*8LS4l^3J7~iIa7{DQ}(TN?#1=l`q?)q#C#8pG*^B)=Dzxs z3bnl{1c5+tXx+F6pWd0%7v>zCpmH=|Lul0;dXaDPI8sQlO4c0G!$WjCR05}#^K?+q z_u)ftSod;7Or)?-$?7eMyx9`ZI<471Bp2zjKlKjw^>$zM&iq^$WuxQpDML-?2~f>t#+Zx~yif zcdLh1#v@0%Oo++;M%M1EYJFB@G^2$Q^CT=|!I-O=f&=$iqVS3XErXI}7roFFOxlb< z-}TmVM6Vj+jPRRHFoT^qQmK;A7stmyCbDF$fCs+8A8;ZVvVK?$54{o{eCH=wW2%9q`w6`2WP6 zy0oNC79r$7ooC6?zU@se{ZT%uYa(HH-E`NvOJ1A=cnG;{Jlb)X7nke z3c13?|7F<-P*&DASK{R-l!|q)Whgmb?NK7AN#pB&xrE_ibVk}jq)?fAh}qhkY0`10 zJ1|`&K=Mg>BKlIa)UqAL%_F>`wu2)L#C+Do=b0%b=#}>R0&3T8X_mOr4gUqUvlg}9 z!EScI+sLFd5KP3tNO;5gX(>3YFwIo(7U{F$eENp?da)9VJDV@1eQxP`)qax5i71^bm$Bb&i5mOm`i0%AzTC;Vl z0LT^rs4Bp~0^$D2uM#~M3IKSn8lW)f&%b{%UB7HA!oO|nZ>IInEUf>@^1lq)pX@)M z!l2OS_V}kP|MF?Sqkb1#{|5bu;(u0M|3Up#bNvqZ-In|U&>;L3@LzZGJK*<0?-$?; v`Y*uW$G*QZ{@3aF#rXPc>;7p&{(V+d6%d}sOaK7!`R#qKmGSey2>|>bSiiej literal 0 HcmV?d00001 diff --git a/scripts/wheels/maturin-2.0.0-py3-none-linux_x86_64.whl b/scripts/wheels/maturin-2.0.0-py3-none-linux_x86_64.whl new file mode 100644 index 0000000000000000000000000000000000000000..aa2cebe88f39f7563dd29dccc7f8563b1e62b808 GIT binary patch literal 5915 zcmZ{obySq?*2V{i5Qb7==pht|p-V~{q-*FHI)+Auk?vHayBSgmk(L$^L{hp-5ReWD zsgL)ZvySKZo%gxde%ABXb>HjR`(FFnKXoM_CJ6ulxC^LHVAV4~G&3{Z0RYl40RTDx zHNepf>HZwy#LmOPb-OGPZb){7leIIaimaBTjHH&NwiYe`h&z^T^;fuifB+bn8$bZy zAFo0!1?L$FlGY2|W-2m=Txx@~fOx4IKV|RS+yS*j!96FQF~hX+ottUWiHOjw2xm1|sw%Jhfh{ z?`KeU2_6~4tBpB0mHZrJE7!=o+@C(eQy9XroV>|XdHj5&~F7iO^ z`zJA;h2VE5Y5diX)4s*fu{9zYGKW4=F=9o9wUni}n*n~R-Z)mjyJsX(n)B1Eytpa{ zorr{9QlhHiFN(XKIaA<=4>RniM&<|v5OPtb_<@#=`Eb$4>U05g)XR=Pnjn*%H5G09g~q>BRgOZb*M)FWz* zt3|(kp1Oj$6Aw0%41=9oB#$l~SJ(Gq($u4)-IAkUpYS0!(QkS?VAU+zO&t9?L;tX z+{WJ(psu{EtTGm$4=xQ?x0AF?j3&KZ-sAYXTz}3^#l%Xc0gb87 zPT?cQ#{{xtonXcB&o15uHuLi}3|RgYH`*+pdON8#ACHE9>hx=k4aJ46dPVgVQ)JYC z0Q~pKe%0QKp2Y?LR!9K=@b8mxA{M>2y7yGk0NK5_*S{cpsKwunFuwh%f_B25x=16hOqHh7eKkvtUyB1z^ zP(JSG3o0Vt&TC42llRlgee-s5OW)`WknEJbVmc1{4seh!M|Z0qePJxlMyC4O=3OCf zud=&so1vH()eoLCBHWccq<1N%M?~aop%mpWEO`&X9u5>|b&Z8j93&GZ%3hv?Ea1W) zEIesmPsgC@eXGG5VaE*}f?uDkY)^`Rv2o^v!{gjnheeeFr-R1I@+?Xx*K4#L2wAn+ zwO<;MKS=HWIQU(vOS)Gb zZh(fHGzLKqz>BsZo~191TCrT7QdOzf{1&q(H5PI$E>RQjiV8fPwHLSfV3Xbg%8$~& zP!;KFKp>r&32U4h5RpVv?$sz=TpkRiw;T9%c!Fnf_;+>;L8l|B?B*KTPV>291t5?@YQGuXljj$=5Q&Id~$c%s1bHSyxzp#}Y#$Z(!MM~Bwn4U22+v0Sb}0 z@QNl?5oY9p!&GF>!}mD!5uu_>Kb()JC869F7BS&j5QKO1RC&DIfu3${bNNL=rr>Vz z5{*`9UbRU&ZGzR2uf>S=+x9rv3Z-Tp%W}tieCcV|?l=ek#$4L8!pJ8lJ*#{IQc%JL z*}R2oi{_vk7ngGMv0oi;u!$R+6^b=&84GUEWskodX6~>x>M?FsR)3+|BU-+u`o(pUoKPy^igB#0?@d(YBKdNEp|A#VR?L#mo*QD*B4F2`w1KVjB$PvZ`e= z;vo@7gdgzTyO07Yx(OiU>G2wH^*rIDxH~n@4jH`VqKWU~6)f9#b&LKog(v$}jS& z6|)u=G#0?zor+t%(KgL!tOapOy5Lrr9ItgwAx|U!YG*e4a;KYUmwifx0KJogAR?m9 zmYJu0i_Y*|#JYS4`v~vFn@05NL6#d)hLIJ)$&C!`>MNAq1X7DOPOqO4oo+ASPpE*S zOmYu!>+RR|vFh=bhQhh6GWA(I2LtT`B?3tz3D6# zSozBeyn~e4na3OB2otc9A*u$9rTA^! zuS|KT9$3Gj(>%pVniF&`G+%e+mO&dSJbHM~=nED0#FY0?isDMClZzgQ*`14}9V__2 zt{xOS!PnT@jYFf$issQ^fz%Ww`%nCNM|5Q* z#oiaAW#6EV9nn+FSDS&~&irl)am`X|9j|E<=0U0gYChnPd!W8qwkNyMEu<@Tl~M9HE*%KqrBVa0U_`UTPf>uYB@&<)|S>jADC{ z9cA|2417=P6|CqJXJ%qyF0FvKhmdbZ)dLe)OpfS?KycpIlI0PjFfS! z%UE8mG{@W8CtUIsoa}(N-QDzr^S|36<9^ye69xby3QIGLE-_ChO| ziZc~Gamx`@&N~RTC7`QRp`~3uCOulBpqeaONkE7Pg0@E@@zaBP5;55V~Vv{6BV(CLu z`&7Bz-vSK|Bd$upBjh=hG{u@Zlu@AYIlm?}V44|-2UN`F+}zB~9ci9EFp!z?34ehVyc}3E zSi7rNKDd)Unmz5Z9ux0#{JowlFS{rcW0Y!4B}3j3eS92r){%FuX&tC`pA{3Edt{>a zT3SgF&>dMSp_g$r{4y}YT2<7t6-?`Fk5bB-tayDeYEtmZ3u+*I?rpeHX1D(2iK0=$Pn$F z=Pf^jObB|9MmsLKRXNdLE_?dc&T$=L|97rWaM3~9H~@eb#hITBwNFrF@@XPTWtQh|f>YoF7>*u-Rm8^`qU zx)R!WQlZS7qYQdlU)NzO-^%-kfK{QbI#l_pu0ncIZ`dpxbY5s?K3ZFef%rhG$TQB> zgpImfN%cKC6IYf8H7=8qY)90Wn=RAbM~vr`r)}zH>5!5qJbKnL@wsGvkS0b8IHHIe zoP~@8R`0%^Cu5D8wG(oxmAdnl(y(;laqnro@m>8=5U)-1Rzpg=@YlRUkxhpItJgL| zsTlSh2y0SaJ0K7TyGoP)o|SqgY1t2m`1yBSX#xSmnlgj@?P<|KH#%UJZvk1O>BC3P z8b*^WueZHtMWIt_272Z?AOcdB(3J=X&EQYeu-W(%0^9DPq>_gO6841zrQWOW&>rn_ zzQjc?8g?}^l~oR2b0Jla>hnCh3$+&9HjXQc`u9uiKR6~X;4fOwcdYH^*Q06L%66e5 zemhb^We%?==?~qeewi70N``sxH0^w@kvKFmXSh_;LJh(#R$XicTOVMHx9!P$EbjV3}qz91|t zE07(^NadP5pm%LjJjOyP&P2ravY$+u(RH2b7$q6@VFXY~NeVvGYcz@^MSW)O@Ci5^5JiM%srl0;IJLmES=g?D_YABSKBLOs8+hGla zs$sG#g5~5bK9teTvMfu9iZs6(kM=ONUYZG{(V*5j-Fc{e*a#NGPLQ8pJ$a@lMyMca z_?Dtf?_@l|hmlPLC8$z@JU_NSNeHj_*dpytrk4R0n8qL7xx@ETpG}X_g+n%?4+O%C+j}rY7(RN~qbr_Pi1gLy7~ad>x>c%tcSiN-fT1C;QRLIA2pZvUN_W~Kd0Ho=kcVNZ= zZzH362u)E+Vz6(dCi{bN2hTas@7@G)Yexr(IH*7alWJd6loa9S^yx{j1~aSM)F%%) ze7^iz`*0`_3p-XX_c##yeW!+O<}fJ{=BS3bjGM9Tun3yMlEE8uFafT}x_zokyI71h zPT!-Q45sBu*I(iReUFpAYEb8Mbe@C%7EVz+YZKyNieMDW12~Ar=_h<~= zC#*qetY?wgmg>Pqki>T1eI_zf6Sk=~BBrzeCf8Klxs0(Fo7}*69WgO^b$Q)Y$>;BK zNrROnrSFDyRhHVL#aa_Ii1vlTU9P6yFLRdOd`{u!`P7zG?89^-&6IiozbnU)(wioJ*6dd>$pg7`5ff8h z*k?LK%!Qp`9X#VGg3R@_ux%Bqj>wNhYp|ChwHi~3&`;|1 ztKPw04e_F_uN+4eV^TjXo+3wTI~l4ces0K4a}rQbZFc*p>SjMyvzA_X!-|Vwy=)q&i_y+d_sw-jKfdK!Gp_1Qb3IMoG8=xTY zudlxYTEB@Z)<22rA2ju^T{!-=%l`spe`){q6a)s|*2mxN@(*nGr_`Th>z_h@Nm1RJ z`+rIO!MXm__;Vcit-*@-TjM{5gFglS^q9W|!1(_u@NdWYp925udi)m1yq!<}?uY#I ZvZyOz-I6^30Q+`z-sVc`mhk}q{|9u}uB-q6 literal 0 HcmV?d00001 diff --git a/scripts/wheels/tqdm-1000.0.0-py3-none-any.whl b/scripts/wheels/tqdm-1000.0.0-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..2b271804ff1479fcafee7900bb6d0bf8ac52ca7b GIT binary patch literal 1017 zcmWIWW@Zs#U|`??VnrZkU|<513=9H5x}-2AS3f>JGcU6wK3=b&l9?d@Mg6m;^C^3Q z>gpL87!;uDbqx&+4D>)SC9}9hH#09SU*Fd?#L>ku#BplC>AXV*0(;(v|6qP)!ssWq z)}erJTqe}5`E?FzCEkQ_C;FW)kmyu#oL%gYrgOGQ2qKl zJ1XSq@fx**tEHd4IoGFBG=Guri@n@oxdu1*bTyU#TsE({ys_$~Wd6!y(P|r({h1zf zc+>2-qX$GHRf)J*lf@Fhq0S@q7H`^zl;ny;K=nC#XTNUad*z11iC{Q7!`7O+!N&L z>>uP(dS>UOCIbeB3-|IpP4ZkWMHH}16ciO=PoA8}di=zZryOzd>!TyA*4Ho7{NmwN z8t8T^@h`(&&Ho#hZSH(~d5PWR3#pbP%%9^P zNGO{8314x}q15tUZ_+8*`w!2hpFNj&I;{KmT8>Kv;=ZDGKT7iI3XjZJ<)5P77rHni z=sQzF)Bb&t9xp#lX8X7`(W`szog4g9Zbvhw)+p@QI&bRk_cj|Ze%s`lcio3UG5-9N zs@GT86Fx8ncr!AIFyl_az>olgC5<2oJw2oAMo-TWy$lRX8i8R2){QT9qnn1FL=dKF r10xCK7Hp=06AYSB7_kdCig7m{qabk|;LXYgQqKZ}%0Sfzz&RWMwNX)m literal 0 HcmV?d00001 diff --git a/scripts/wheels/tqdm-4.66.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl b/scripts/wheels/tqdm-4.66.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl new file mode 100644 index 0000000000000000000000000000000000000000..62d54fe3beaad40007f29b03b1f51cfaf924e589 GIT binary patch literal 5579 zcmZ{o2Q*ymy2nRnj3^~}wYq{l zY#kG4)K}ZPt>@NLQpixK!m6a|BQ35;TF(HkNPFnOxdiY|fIgIPmU_Y(N9enCD%`5M z%hJbZ#Q_t#PN{k@U%hxGu?93L~^h* zJmA&H`5N*lLK`C9K+~>(m%K6E-R9xSsO@#;dz=1_Q*Z(&OUQ)qoyySc$VCo&rItrt zjtF#dGY`M8N?5|IV5UpXo_<7d=s}`&)YD7S==J)ntq=)?v|l#7j8wzN7F zLyh!4pn~o}elW^w2eGS37?&27HVc;;geM#GJ}4+NQ7#E{3MB6He5zbR+q+Mk`|f>K z)^r@-aV^5%O!m&fyp1Kx2z;PM{vpX?tflpQ-oQR59Li-yQN~WQ$ue|OI+}VgYl?Tb z!1;yz1ENq`JlR>NZHb@X_q!3NezCQ(>&MRC8AuIx@z7d;cJrY!;Rg|y)3TsV%0W_L zrmFb>W0OX?an%6CQ0$L38QJ83N{HX~qj~N}v1whmf>!U7&ArR!9y>K=;X}Vw+BI^_ zakn95$*%iBLXh`D>;Lx3)b(BgPp3Ak;sO921ONczzxE2R5FZ4>caP7?*4=~G*2%`1 zUrkw7UPD$Mp=aj&U4ha+8|I2=Ywv$8q|fR&&mizvuQJgQB446EIfauo0bfUouw5KO zE!!a5Cpm;ct4s6lJ56gthUZV4&Q(UVH2li?>nrmidWFqHlPYTlZG>W+7vZQBb>5ge z*4#OyBZT)M4Y^00754p)9&tGqJ`IsNa=ObqZfAd%A3A6q+~v` z@15XD>>zxHcydrkEu5TAME;$AUc3G+pID1PsK7}OgknW|V%`pC1%-`YRj>O*s*VZ>m2I-Kv*0GAO=%JgQ~ z$0pc|t5twDXZwzLCMr54;1qrps)uu;Z|-awp7m-vS7x5_3K5B!cnOU(Ji&&hWYrSCcpJ z%&L;vjkyVm-mSjhDNcSs`^^HBsjG=qe(2E`D@chDNJ!b5Xj%{4bj5O^p(9wULf94c z(d%pt?G^`JpykWw$iEu;ip!1MSqN(zC2AG<5-1|Eb@hm-M`7*jhS)@1x%7ucamZg|X1aBG;Q~ z^m)Jyn>Q-@wc=ka_886A#5P59oJMrJ)IZ{I;h|aSSBHia-`Ew`Iql?FExN%Q^~M}G zL^9*;BPW+47CJxllO$D79QW@ZJ$tXx#F(aaFj7oOar$O;e z0W$~*SM1%d?g=4Xx+ZM&5y$s1g?{8q z5>V&Tpb1a)iGVW2+6>%_&sryU)dm^5PthvZyk=P7q9WBzbztJi<4WmZ&E9ud`4aM! z+^t$UQ%fqE`Sa;BQDO=co@DvVu)y1_j$L`m*sole6w*lb(AuX/JQ944gkR4X;b z0`lNhDyO4DdA1Vt;Zx5+9X#;N&@t(p!Y@1v?(sbx5^y&(MN}gDM){j5sHSW~cOdnG zct4z>>Ar&ND|RE_!uy~3a!?W^P#{H?m^6oS%#ntM+yXFOiaV;)^44A9)-A&F#^d`W z6{bCp1!T*|^nY$!XJ3%~6NO5W!A6hm&2C^_N1=N+001Tc!(SQeKckSkyq>h2w4SuS z9uWX%e>BVb*YJcB1F&${aR7k7E^iDSlcynMmqQi9%L8UM0wJn?wsYnMxH~q_7A>BX z4>rVAQfno$D<=7DRG`Xzh{6P?ez;`vKv9;q-gO_C0+-0WXV2+ls7!vK%fA#u4{4sS zwkf!73tZ_gT{5M<6pCy5dWw#GWcW1Ro}fP}5*Kpc*QVVB0w5@WEXaud#95L?hR$wQ z5vDMEzAQ$2Gt4}ER!=?h+r&`+CRCe)-*<*fStqrv*78Zhf zJG45)KH&@z3`%o3^cW!%i-#mUS42^@lMY(;IM+0%4I#zwS ztH@Hnjj}0i^G}u5MwX(I-VAcgj$xKwBfa6pqa$AZE(D%OhS0HoJ0j1_n9%RlJ9419 zxaEncBEVA3&P4w`&R*NW5xgN*-WLt>htqijThOA?kt~bevEX1j>Z?`Y*A8Wvqu)(c)jhhBQ5jO$z=e89jm))m zt7-(dc!xjJ7fMavZfGX0bSun-KSwaj@{20yCc8i9q5k`ZhQML(MI8C zzjR_3xbV9VK*0TTMm&As3B`|xLAA>YrVsa`DSWm|L-s^CjBdNTL`6!DKK%G0HAl!7 zM)mDfD{WuZx~Bp`Tjo8^RaX28eFknS>f7p$#8v58Q!WRKW^{~Q%Cn*- zdnu}v798*Sl8ajvV+`Rdg%K>=nRMoUnwcM0UGtV_R`<12gLUeh*K}(GJ6n&%Mq5i9 zgU0*U>KWNO(Tpi|c+AD5gHOxYZ2V^B>-WTj+D+@eMNFE#bGkYC5v@8SuS@g0zIV~Q}Z!8{Vq$#y5?5uK2rcRO*geXI(9Hv zpUwm4QXQAGCm8>{<>1k>MM0bb%S`A@%#-{ysb_)OaNORhi6?z^b00fIfz3mVlwJnS zo%vv%?I;LfKWXaM)9in^10{+izA#_gZ#q0RtBS^o@UKgut6+Zmu?lxkc%0*M(h0S< z*FKNlT&A}xPK7&`hQp}C&W{FN!1Ar>o{gXUC~;mUc9F0+n}_@6n5Ts5h?nUa-fD~6 z+hBO^3w+0@R%@PF9AQ;M@dcC#6(7oSbz_8%GF~PBAU_Xlg%Q4Rc2`8`0;AgdJ9ZoK=`U+54 ztwD#gP-tRs0Hr#vjx1UJjt%&CQ`Om9>KxZ*?p}{y_g{SepV7|R$-~VD=HhJY|cti7xqel5d&ExPi%RctW3E8`pYmdf5}7UZ?x-5 zdvvl5pE#+zpqjLuYHNr3guC0!F6WmjJNM3vsL6mIUkhvzU}!y^HFVf(D>ZDzg{+hf zzs6L7SAfLMw@30SQ`>W^5)Ca3+NB83+ydkd0}8vJbLg+KtcJL%1wU#b*-{eeO?BF* z?~sg8r*j_xVlt#Jf`uNrrDKdrpUm#3zbU1l&2rTUD>6=5ezUL>8Qzv`RJ-2$s4_hc zOO{i5lz+V!qptlVl%BoQ`P^O>Wo%4>b|eSyNsQfU*zHFa>@gs`gyUP~50|z$EKl$| z3Td>e8;~P{{kg1d!QK(ReWwOK8Xf`hQ5-3y+@D=_tE893^&ZGBKTzK6 z?Vi*lSXy++;yzNW9IG-~4boc@)W4uQ0HALi<{3ObF(LgaB8m@WLqIC>EoOBezXLV} zktKD(hD2Ieg2XRvTwExLA8mGCt z|7F04H`YJvTf^lrLm}&Tqnl!z6!PA;Do(4U)u3`qUEd6}+EN3eX?9oawS4Wil0GpG zHfFR6FYKuFx#pK_${BbAwLWtN0%As|h;57OW5P;{KAs)T^eda33>HII%K=hMuqP)I zj;(Oxm6^KnhPIYXRKQ{^Mes8;)sVq1hLyHH$cuX}ud5heo9%U~U4x>0<=sy{R=wdw zKZT-EOS&%1fw>hsKdy5a)M_j6c6;r3hQyj*aGhbi7!e`>^}*jQ%{JTs+bvaqOrxV> z1Dv~4b1W9eICeIxjS(KJm@Y0liKX7()5GY~Q=i|I`Eq_7a|jK|IaU3kjCjQpH=L+g z_g=r@Wz76it#|8Lm{EFE1EtsP)t9LQ0qKjp-9neeKB;&c88T_+h_B-BYONOYHMiCu zAo}K_=oi)Jn_2ne&2+MX%AF0e9Sg$13$ixaE!X{U~|?&kT^SYKP}~4S>ek-KUlO7-(0#f9Hd1i z%~FAV;praa{7D=f{f;Uk@=aYI<2K$Gfz?-!?#-@;*nD*<6M5j(J3UIy!nKK*zS8gn zmo&Ufwam~w>>INcYVn*=kve<6dLzk{&oya^B{~yMj2)N@`|x-McoE-rK|LMRJ+wqV zzQCzmaWN9jqSkXsX9KFBTeJhkorw66;yh-il>d15EmJe{>gy)4-nbwSE2wsMa^}gz z4e}Hpo96X511LR}!e*f}lQZUdL9P0<-~W$l$`WvG!ZR{E{w2$4yFP&*jD8*_uB z!p*+-sU+GDga&)lwzX0`iQ|!VFCtiFGKyy+0LqGM^X)n*lIbz?wmN0@v;_=C)O_;w2EKx7gsV9%lNA0P(@@X5!)I#ql0k4kEMu_L{2D-ek)mb2b`CGIFC6^tkFMN-|e5X z6iPPD5^#Lv@g}0N%{0NXj?97Qh?h6!RtOHwCGHg+vA13G_oXIQ6}kYEAfi=kswl$5 zmhh0HIF@;ud0RR*JARRxO-`zNWK!n#@Nj?fF6z^>{pZuxp0hv88APR5LYzI7oH+uM zEe#+eMG1~j=_hIj78X-=MukWO>z5!2+MhAffdSSBSDVU>0ut_z+l+aZcg;O?!G~l| z4?2iCdlaXXKh_J58Xc7mXl$8`C!3sYKVb==E#n2gJc#|?>M3|rtn+6K4}ZTg2rNUL zxe)`4^K;R@i1wL-9ZCVQBNPJeI3wT(Ls}-;7?~ zpo^{zAB~M}P;aT+Sd1ASnmGK}9$w%%uuhBEtuD4o$?wsZ62xiu)2WVe@Rih%ak4(b zn$#f3E72G#PQSZu_(K?X$6=4`XeL`r6$_gZ=g%Fv>lFIekstKy_IrQs|Ihw!J(|Dg z0s#3zJlKEA#sBKq{E}k&my!Q0v}pcPy6zNSOI73lN$Q`n3-9l;>$j5Yce}qc*FScN zukqo(!_Hp>_PfyUob->-+dm5ZPEEgS{!T>yXaYfhGWTzM^t<5iX!O5=wLv<66#N^M Xv{dn~zc~OvaDB30|Hy=!zn=aFET#8f literal 0 HcmV?d00001