From 34341bd6e9baff45342b75cf24861ce84725f8c4 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Wed, 3 Apr 2024 19:23:37 -0400 Subject: [PATCH] Allow package lookups across multiple indexes via explicit opt-in (#2815) ## Summary This partially revives https://github.com/astral-sh/uv/pull/2135 (with some modifications) to enable users to opt-in to looking for packages across multiple indexes. The behavior is such that, in version selection, we take _any_ compatible version from a "higher-priority" index over the compatible versions of a "lower-priority" index, even if that means we might accept an "older" version. Closes https://github.com/astral-sh/uv/issues/2775. --- Cargo.lock | 2 + PIP_COMPATIBILITY.md | 6 ++ README.md | 3 + crates/uv-client/Cargo.toml | 1 + crates/uv-client/src/registry_client.rs | 46 +++++--- crates/uv-dev/src/resolve_many.rs | 15 ++- crates/uv-resolver/src/candidate_selector.rs | 55 ++++++---- crates/uv-resolver/src/error.rs | 17 +-- crates/uv-resolver/src/resolution.rs | 28 ++--- crates/uv-resolver/src/resolver/mod.rs | 17 ++- crates/uv-resolver/src/resolver/provider.rs | 39 ++++--- crates/uv-resolver/src/version_map.rs | 12 +-- crates/uv-types/Cargo.toml | 1 + crates/uv-types/src/build_options.rs | 23 ++++ crates/uv/Cargo.toml | 2 +- crates/uv/src/commands/pip_compile.rs | 6 +- crates/uv/src/commands/pip_install.rs | 6 +- crates/uv/src/commands/pip_sync.rs | 6 +- crates/uv/src/commands/venv.rs | 7 +- crates/uv/src/logging.rs | 2 +- crates/uv/src/main.rs | 42 +++++++- crates/uv/tests/common/mod.rs | 14 ++- crates/uv/tests/pip_compile.rs | 107 +++++++++++++++++++ 23 files changed, 351 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8cba8cb4..0395e2662 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4464,6 +4464,7 @@ dependencies = [ "uv-cache", "uv-fs", "uv-normalize", + "uv-types", "uv-version", "uv-warnings", "webpki-roots", @@ -4785,6 +4786,7 @@ name = "uv-types" version = "0.0.1" dependencies = [ "anyhow", + "clap", "distribution-types", "itertools 0.12.1", "once-map", diff --git a/PIP_COMPATIBILITY.md b/PIP_COMPATIBILITY.md index 413f117c6..77bde9115 100644 --- a/PIP_COMPATIBILITY.md +++ b/PIP_COMPATIBILITY.md @@ -128,6 +128,12 @@ internal package, thus causing the malicious package to be installed instead of package. See, for example, [the `torchtriton` attack](https://pytorch.org/blog/compromised-nightly-dependency/) from December 2022. +As of v0.1.29, users can opt in to `pip`-style behavior for multiple indexes via the +`--index-strategy unsafe-any-match` command-line option, or the `UV_INDEX_STRATEGY` environment +variable. When enabled, uv will search for each package across all indexes, and consider all +available versions when resolving dependencies, prioritizing the `--extra-index-url` indexes over +the default index URL. (Versions that are duplicated _across_ indexes will be ignored.) + In the future, uv will support pinning packages to dedicated indexes (see: [#171](https://github.com/astral-sh/uv/issues/171)). Additionally, [PEP 708](https://peps.python.org/pep-0708/) is a provisional standard that aims to address the "dependency confusion" issue across package registries and installers. diff --git a/README.md b/README.md index 8bba8c9fb..a00d547bb 100644 --- a/README.md +++ b/README.md @@ -449,6 +449,9 @@ uv accepts the following command-line arguments as environment variables: should be used with caution, as it can modify the system Python installation. - `UV_NATIVE_TLS`: Equivalent to the `--native-tls` command-line argument. If set to `true`, uv will use the system's trust store instead of the bundled `webpki-roots` crate. +- `UV_INDEX_STRATEGY`: Equivalent to the `--index-strategy` command-line argument. For example, if + set to `unsafe-any-match`, uv will consider versions of a given package available across all + index URLs, rather than limiting its search to the first index URL that contains the package. In each case, the corresponding command-line argument takes precedence over an environment variable. diff --git a/crates/uv-client/Cargo.toml b/crates/uv-client/Cargo.toml index 0d89d290b..8bcdf8e5d 100644 --- a/crates/uv-client/Cargo.toml +++ b/crates/uv-client/Cargo.toml @@ -15,6 +15,7 @@ uv-auth = { workspace = true } uv-cache = { workspace = true } uv-fs = { workspace = true, features = ["tokio"] } uv-normalize = { workspace = true } +uv-types = { workspace = true } uv-version = { workspace = true } uv-warnings = { workspace = true } pypi-types = { workspace = true } diff --git a/crates/uv-client/src/registry_client.rs b/crates/uv-client/src/registry_client.rs index 8ae51f6b2..ec9bbf572 100644 --- a/crates/uv-client/src/registry_client.rs +++ b/crates/uv-client/src/registry_client.rs @@ -23,6 +23,7 @@ use pypi_types::{Metadata23, SimpleJson}; use uv_auth::KeyringProvider; use uv_cache::{Cache, CacheBucket, WheelCache}; use uv_normalize::PackageName; +use uv_types::IndexStrategy; use crate::base_client::{BaseClient, BaseClientBuilder}; use crate::cached_client::CacheControl; @@ -35,6 +36,7 @@ use crate::{CachedClient, CachedClientError, Error, ErrorKind}; #[derive(Debug, Clone)] pub struct RegistryClientBuilder<'a> { index_urls: IndexUrls, + index_strategy: IndexStrategy, keyring_provider: KeyringProvider, native_tls: bool, retries: u32, @@ -49,6 +51,7 @@ impl RegistryClientBuilder<'_> { pub fn new(cache: Cache) -> Self { Self { index_urls: IndexUrls::default(), + index_strategy: IndexStrategy::default(), keyring_provider: KeyringProvider::default(), native_tls: false, cache, @@ -68,6 +71,12 @@ impl<'a> RegistryClientBuilder<'a> { self } + #[must_use] + pub fn index_strategy(mut self, index_strategy: IndexStrategy) -> Self { + self.index_strategy = index_strategy; + self + } + #[must_use] pub fn keyring_provider(mut self, keyring_provider: KeyringProvider) -> Self { self.keyring_provider = keyring_provider; @@ -147,6 +156,7 @@ impl<'a> RegistryClientBuilder<'a> { RegistryClient { index_urls: self.index_urls, + index_strategy: self.index_strategy, cache: self.cache, connectivity, client, @@ -160,6 +170,8 @@ impl<'a> RegistryClientBuilder<'a> { pub struct RegistryClient { /// The index URLs to use for fetching packages. index_urls: IndexUrls, + /// The strategy to use when fetching across multiple indexes. + index_strategy: IndexStrategy, /// The underlying HTTP client. client: CachedClient, /// Used for the remote wheel METADATA cache. @@ -206,17 +218,23 @@ impl RegistryClient { pub async fn simple( &self, package_name: &PackageName, - ) -> Result<(IndexUrl, OwnedArchive), Error> { + ) -> Result)>, Error> { let mut it = self.index_urls.indexes().peekable(); if it.peek().is_none() { return Err(ErrorKind::NoIndex(package_name.as_ref().to_string()).into()); } + let mut results = Vec::new(); for index in it { - let result = self.simple_single_index(package_name, index).await?; + match self.simple_single_index(package_name, index).await? { + Ok(metadata) => { + results.push((index.clone(), metadata)); - return match result { - Ok(metadata) => Ok((index.clone(), metadata)), + // If we're only using the first match, we can stop here. + if self.index_strategy == IndexStrategy::FirstMatch { + break; + } + } Err(CachedClientError::Client(err)) => match err.into_kind() { ErrorKind::Offline(_) => continue, ErrorKind::ReqwestError(err) => { @@ -225,20 +243,24 @@ impl RegistryClient { { continue; } - Err(ErrorKind::from(err).into()) + return Err(ErrorKind::from(err).into()); } - other => Err(other.into()), + other => return Err(other.into()), }, - Err(CachedClientError::Callback(err)) => Err(err), + Err(CachedClientError::Callback(err)) => return Err(err), }; } - match self.connectivity { - Connectivity::Online => { - Err(ErrorKind::PackageNotFound(package_name.to_string()).into()) - } - Connectivity::Offline => Err(ErrorKind::Offline(package_name.to_string()).into()), + if results.is_empty() { + return match self.connectivity { + Connectivity::Online => { + Err(ErrorKind::PackageNotFound(package_name.to_string()).into()) + } + Connectivity::Offline => Err(ErrorKind::Offline(package_name.to_string()).into()), + }; } + + Ok(results) } async fn simple_single_index( diff --git a/crates/uv-dev/src/resolve_many.rs b/crates/uv-dev/src/resolve_many.rs index f5608b46a..bd0e53b1c 100644 --- a/crates/uv-dev/src/resolve_many.rs +++ b/crates/uv-dev/src/resolve_many.rs @@ -47,10 +47,17 @@ async fn find_latest_version( client: &RegistryClient, package_name: &PackageName, ) -> Option { - let (_, raw_simple_metadata) = client.simple(package_name).await.ok()?; - let simple_metadata = OwnedArchive::deserialize(&raw_simple_metadata); - let version = simple_metadata.into_iter().next()?.version; - Some(version) + client + .simple(package_name) + .await + .ok() + .into_iter() + .flatten() + .filter_map(|(_index, raw_simple_metadata)| { + let simple_metadata = OwnedArchive::deserialize(&raw_simple_metadata); + Some(simple_metadata.into_iter().next()?.version) + }) + .max() } pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> { diff --git a/crates/uv-resolver/src/candidate_selector.rs b/crates/uv-resolver/src/candidate_selector.rs index eff292bd9..33a20d73d 100644 --- a/crates/uv-resolver/src/candidate_selector.rs +++ b/crates/uv-resolver/src/candidate_selector.rs @@ -71,7 +71,7 @@ impl CandidateSelector { &'a self, package_name: &'a PackageName, range: &'a Range, - version_map: &'a VersionMap, + version_maps: &'a [VersionMap], preferences: &'a Preferences, installed_packages: &'a InstalledPackages, exclusions: &'a Exclusions, @@ -107,7 +107,10 @@ impl CandidateSelector { } // Check for a remote distribution that matches the preferred version - if let Some(file) = version_map.get(version) { + if let Some(file) = version_maps + .iter() + .find_map(|version_map| version_map.get(version)) + { return Some(Candidate::new(package_name, version, file)); } } @@ -163,33 +166,39 @@ impl CandidateSelector { "selecting candidate for package {:?} with range {:?} with {} remote versions", package_name, range, - version_map.len() + version_maps.iter().map(VersionMap::len).sum::(), ); match &self.resolution_strategy { - ResolutionStrategy::Highest => Self::select_candidate( - version_map.iter().rev(), - package_name, - range, - allow_prerelease, - ), - ResolutionStrategy::Lowest => { + ResolutionStrategy::Highest => version_maps.iter().find_map(|version_map| { + Self::select_candidate( + version_map.iter().rev(), + package_name, + range, + allow_prerelease, + ) + }), + ResolutionStrategy::Lowest => version_maps.iter().find_map(|version_map| { Self::select_candidate(version_map.iter(), package_name, range, allow_prerelease) - } + }), ResolutionStrategy::LowestDirect(direct_dependencies) => { if direct_dependencies.contains(package_name) { - Self::select_candidate( - version_map.iter(), - package_name, - range, - allow_prerelease, - ) + version_maps.iter().find_map(|version_map| { + Self::select_candidate( + version_map.iter(), + package_name, + range, + allow_prerelease, + ) + }) } else { - Self::select_candidate( - version_map.iter().rev(), - package_name, - range, - allow_prerelease, - ) + version_maps.iter().find_map(|version_map| { + Self::select_candidate( + version_map.iter().rev(), + package_name, + range, + allow_prerelease, + ) + }) } } } diff --git a/crates/uv-resolver/src/error.rs b/crates/uv-resolver/src/error.rs index 6cf92bf4c..96c46fcb1 100644 --- a/crates/uv-resolver/src/error.rs +++ b/crates/uv-resolver/src/error.rs @@ -209,14 +209,15 @@ impl NoSolutionError { // we represent the state of the resolver at the time of failure. if visited.contains(name) { if let Some(response) = package_versions.get(name) { - if let VersionsResponse::Found(ref version_map) = *response { - available_versions.insert( - package.clone(), - version_map - .iter() - .map(|(version, _)| version.clone()) - .collect(), - ); + if let VersionsResponse::Found(ref version_maps) = *response { + for version_map in version_maps { + available_versions + .entry(package.clone()) + .or_insert_with(BTreeSet::new) + .extend( + version_map.iter().map(|(version, _)| version.clone()), + ); + } } } } diff --git a/crates/uv-resolver/src/resolution.rs b/crates/uv-resolver/src/resolution.rs index 667f79154..b1c29d34f 100644 --- a/crates/uv-resolver/src/resolution.rs +++ b/crates/uv-resolver/src/resolution.rs @@ -99,12 +99,14 @@ impl ResolutionGraph { if let Some(hash) = preferences.match_hashes(package_name, version) { hashes.insert(package_name.clone(), hash.to_vec()); } else if let Some(versions_response) = packages.get(package_name) { - if let VersionsResponse::Found(ref version_map) = *versions_response { - hashes.insert(package_name.clone(), { - let mut hash = version_map.hashes(version); - hash.sort_unstable(); - hash - }); + if let VersionsResponse::Found(ref version_maps) = *versions_response { + for version_map in version_maps { + if let Some(mut hash) = version_map.hashes(version) { + hash.sort_unstable(); + hashes.insert(package_name.clone(), hash); + break; + } + } } } @@ -127,12 +129,14 @@ impl ResolutionGraph { if let Some(hash) = preferences.match_hashes(package_name, version) { hashes.insert(package_name.clone(), hash.to_vec()); } else if let Some(versions_response) = packages.get(package_name) { - if let VersionsResponse::Found(ref version_map) = *versions_response { - hashes.insert(package_name.clone(), { - let mut hash = version_map.hashes(version); - hash.sort_unstable(); - hash - }); + if let VersionsResponse::Found(ref version_maps) = *versions_response { + for version_map in version_maps { + if let Some(mut hash) = version_map.hashes(version) { + hash.sort_unstable(); + hashes.insert(package_name.clone(), hash); + break; + } + } } } diff --git a/crates/uv-resolver/src/resolver/mod.rs b/crates/uv-resolver/src/resolver/mod.rs index 5b99e307d..379a18c0c 100644 --- a/crates/uv-resolver/src/resolver/mod.rs +++ b/crates/uv-resolver/src/resolver/mod.rs @@ -6,7 +6,6 @@ use std::ops::Deref; use std::sync::Arc; use anyhow::Result; - use dashmap::{DashMap, DashSet}; use futures::{FutureExt, StreamExt}; use itertools::Itertools; @@ -34,7 +33,6 @@ use uv_normalize::PackageName; use uv_types::{BuildContext, Constraints, InstalledPackagesProvider, Overrides}; use crate::candidate_selector::{CandidateDist, CandidateSelector}; - use crate::editables::Editables; use crate::error::ResolveError; use crate::manifest::Manifest; @@ -54,7 +52,7 @@ pub use crate::resolver::provider::{ use crate::resolver::reporter::Facade; pub use crate::resolver::reporter::{BuildId, Reporter}; use crate::yanks::AllowedYanks; -use crate::{DependencyMode, Exclusions, Options, VersionMap}; +use crate::{DependencyMode, Exclusions, Options}; mod index; mod locals; @@ -632,23 +630,22 @@ impl< .ok_or(ResolveError::Unregistered)?; self.visited.insert(package_name.clone()); - let empty_version_map = VersionMap::default(); - let version_map = match *versions_response { - VersionsResponse::Found(ref version_map) => version_map, + let version_maps = match *versions_response { + VersionsResponse::Found(ref version_maps) => version_maps.as_slice(), VersionsResponse::NoIndex => { self.unavailable_packages .insert(package_name.clone(), UnavailablePackage::NoIndex); - &empty_version_map + &[] } VersionsResponse::Offline => { self.unavailable_packages .insert(package_name.clone(), UnavailablePackage::Offline); - &empty_version_map + &[] } VersionsResponse::NotFound => { self.unavailable_packages .insert(package_name.clone(), UnavailablePackage::NotFound); - &empty_version_map + &[] } }; @@ -664,7 +661,7 @@ impl< let Some(candidate) = self.selector.select( package_name, range, - version_map, + version_maps, &self.preferences, self.installed_packages, &self.exclusions, diff --git a/crates/uv-resolver/src/resolver/provider.rs b/crates/uv-resolver/src/resolver/provider.rs index 266f7fc1e..273fc2b53 100644 --- a/crates/uv-resolver/src/resolver/provider.rs +++ b/crates/uv-resolver/src/resolver/provider.rs @@ -22,7 +22,7 @@ pub type WheelMetadataResult = Result; #[derive(Debug)] pub enum VersionsResponse { /// The package was found in the registry with the included versions - Found(VersionMap), + Found(Vec), /// The package was not found in the registry NotFound, /// The package was not found in the local registry @@ -113,29 +113,36 @@ impl<'a, Context: BuildContext + Send + Sync> ResolverProvider // If the "Simple API" request was successful, convert to `VersionMap` on the Tokio // threadpool, since it can be slow. match result { - Ok((index, metadata)) => Ok(VersionsResponse::Found(VersionMap::from_metadata( - metadata, - package_name, - &index, - &self.tags, - &self.python_requirement, - &self.allowed_yanks, - self.exclude_newer.as_ref(), - self.flat_index.get(package_name).cloned(), - &self.no_binary, - &self.no_build, - ))), + Ok(results) => Ok(VersionsResponse::Found( + results + .into_iter() + .map(|(index, metadata)| { + VersionMap::from_metadata( + metadata, + package_name, + &index, + &self.tags, + &self.python_requirement, + &self.allowed_yanks, + self.exclude_newer.as_ref(), + self.flat_index.get(package_name).cloned(), + &self.no_binary, + &self.no_build, + ) + }) + .collect(), + )), Err(err) => match err.into_kind() { uv_client::ErrorKind::PackageNotFound(_) => { if let Some(flat_index) = self.flat_index.get(package_name).cloned() { - Ok(VersionsResponse::Found(VersionMap::from(flat_index))) + Ok(VersionsResponse::Found(vec![VersionMap::from(flat_index)])) } else { Ok(VersionsResponse::NotFound) } } uv_client::ErrorKind::NoIndex(_) => { if let Some(flat_index) = self.flat_index.get(package_name).cloned() { - Ok(VersionsResponse::Found(VersionMap::from(flat_index))) + Ok(VersionsResponse::Found(vec![VersionMap::from(flat_index)])) } else if self.flat_index.offline() { Ok(VersionsResponse::Offline) } else { @@ -144,7 +151,7 @@ impl<'a, Context: BuildContext + Send + Sync> ResolverProvider } uv_client::ErrorKind::Offline(_) => { if let Some(flat_index) = self.flat_index.get(package_name).cloned() { - Ok(VersionsResponse::Found(VersionMap::from(flat_index))) + Ok(VersionsResponse::Found(vec![VersionMap::from(flat_index)])) } else { Ok(VersionsResponse::Offline) } diff --git a/crates/uv-resolver/src/version_map.rs b/crates/uv-resolver/src/version_map.rs index b3045e600..c7937bef8 100644 --- a/crates/uv-resolver/src/version_map.rs +++ b/crates/uv-resolver/src/version_map.rs @@ -174,16 +174,10 @@ impl VersionMap { } /// Return the [`Hashes`] for the given version, if any. - pub(crate) fn hashes(&self, version: &Version) -> Vec { + pub(crate) fn hashes(&self, version: &Version) -> Option> { match self.inner { - VersionMapInner::Eager(ref map) => map - .get(version) - .map(|file| file.hashes().to_vec()) - .unwrap_or_default(), - VersionMapInner::Lazy(ref lazy) => lazy - .get(version) - .map(|file| file.hashes().to_vec()) - .unwrap_or_default(), + VersionMapInner::Eager(ref map) => map.get(version).map(|file| file.hashes().to_vec()), + VersionMapInner::Lazy(ref lazy) => lazy.get(version).map(|file| file.hashes().to_vec()), } } diff --git a/crates/uv-types/Cargo.toml b/crates/uv-types/Cargo.toml index e6210fb49..4fe6ccb29 100644 --- a/crates/uv-types/Cargo.toml +++ b/crates/uv-types/Cargo.toml @@ -21,6 +21,7 @@ uv-interpreter = { workspace = true } uv-normalize = { workspace = true } anyhow = { workspace = true } +clap = { workspace = true, features = ["derive"], optional = true } itertools = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true, optional = true } diff --git a/crates/uv-types/src/build_options.rs b/crates/uv-types/src/build_options.rs index 81b646576..bcdaee92b 100644 --- a/crates/uv-types/src/build_options.rs +++ b/crates/uv-types/src/build_options.rs @@ -211,6 +211,29 @@ impl NoBuild { } } +#[derive(Debug, Default, Clone, Hash, Eq, PartialEq)] +#[cfg_attr(feature = "clap", derive(clap::ValueEnum))] +pub enum IndexStrategy { + /// Only use results from the first index that returns a match for a given package name. + /// + /// While this differs from pip's behavior, it's the default index strategy as it's the most + /// secure. + #[default] + FirstMatch, + /// Search for every package name across all indexes, exhausting the versions from the first + /// index before moving on to the next. + /// + /// In this strategy, we look for every package across all indexes. When resolving, we attempt + /// to use versions from the indexes in order, such that we exhaust all available versions from + /// the first index before moving on to the next. Further, if a version is found to be + /// incompatible in the first index, we do not reconsider that version in subsequent indexes, + /// even if the secondary index might contain compatible versions (e.g., variants of the same + /// versions with different ABI tags or Python version constraints). + /// + /// See: https://peps.python.org/pep-0708/ + UnsafeAnyMatch, +} + #[cfg(test)] mod tests { use std::str::FromStr; diff --git a/crates/uv/Cargo.toml b/crates/uv/Cargo.toml index 1dba92d0a..2fde36bcf 100644 --- a/crates/uv/Cargo.toml +++ b/crates/uv/Cargo.toml @@ -31,7 +31,7 @@ uv-interpreter = { workspace = true } uv-normalize = { workspace = true } uv-requirements = { workspace = true } uv-resolver = { workspace = true, features = ["clap"] } -uv-types = { workspace = true } +uv-types = { workspace = true, features = ["clap"] } uv-virtualenv = { workspace = true } uv-warnings = { workspace = true } diff --git a/crates/uv/src/commands/pip_compile.rs b/crates/uv/src/commands/pip_compile.rs index 28c28978c..31681e883 100644 --- a/crates/uv/src/commands/pip_compile.rs +++ b/crates/uv/src/commands/pip_compile.rs @@ -36,8 +36,8 @@ use uv_resolver::{ OptionsBuilder, PreReleaseMode, PythonRequirement, ResolutionMode, Resolver, }; use uv_types::{ - BuildIsolation, ConfigSettings, Constraints, EmptyInstalledPackages, InFlight, NoBinary, - NoBuild, Overrides, SetupPyStrategy, Upgrade, + BuildIsolation, ConfigSettings, Constraints, EmptyInstalledPackages, InFlight, IndexStrategy, + NoBinary, NoBuild, Overrides, SetupPyStrategy, Upgrade, }; use uv_warnings::warn_user; @@ -67,6 +67,7 @@ pub(crate) async fn pip_compile( include_find_links: bool, include_marker_expression: bool, index_locations: IndexLocations, + index_strategy: IndexStrategy, keyring_provider: KeyringProvider, setup_py: SetupPyStrategy, config_settings: ConfigSettings, @@ -210,6 +211,7 @@ pub(crate) async fn pip_compile( .native_tls(native_tls) .connectivity(connectivity) .index_urls(index_locations.index_urls()) + .index_strategy(index_strategy) .keyring_provider(keyring_provider) .markers(&markers) .platform(interpreter.platform()) diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 0dc96ad05..ba11f3f7a 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -38,8 +38,8 @@ use uv_resolver::{ Preference, ResolutionGraph, ResolutionMode, Resolver, }; use uv_types::{ - BuildIsolation, ConfigSettings, Constraints, InFlight, NoBinary, NoBuild, Overrides, Reinstall, - SetupPyStrategy, Upgrade, + BuildIsolation, ConfigSettings, Constraints, InFlight, IndexStrategy, NoBinary, NoBuild, + Overrides, Reinstall, SetupPyStrategy, Upgrade, }; use uv_warnings::warn_user; @@ -61,6 +61,7 @@ pub(crate) async fn pip_install( dependency_mode: DependencyMode, upgrade: Upgrade, index_locations: IndexLocations, + index_strategy: IndexStrategy, keyring_provider: KeyringProvider, reinstall: Reinstall, link_mode: LinkMode, @@ -195,6 +196,7 @@ pub(crate) async fn pip_install( .native_tls(native_tls) .connectivity(connectivity) .index_urls(index_locations.index_urls()) + .index_strategy(index_strategy) .keyring_provider(keyring_provider) .markers(markers) .platform(interpreter.platform()) diff --git a/crates/uv/src/commands/pip_sync.rs b/crates/uv/src/commands/pip_sync.rs index acc8ac8df..c5f11d73f 100644 --- a/crates/uv/src/commands/pip_sync.rs +++ b/crates/uv/src/commands/pip_sync.rs @@ -28,8 +28,8 @@ use uv_requirements::{ }; use uv_resolver::{DependencyMode, InMemoryIndex, Manifest, OptionsBuilder, Resolver}; use uv_types::{ - BuildIsolation, ConfigSettings, EmptyInstalledPackages, InFlight, NoBinary, NoBuild, Reinstall, - SetupPyStrategy, + BuildIsolation, ConfigSettings, EmptyInstalledPackages, InFlight, IndexStrategy, NoBinary, + NoBuild, Reinstall, SetupPyStrategy, }; use uv_warnings::warn_user; @@ -45,6 +45,7 @@ pub(crate) async fn pip_sync( link_mode: LinkMode, compile: bool, index_locations: IndexLocations, + index_strategy: IndexStrategy, keyring_provider: KeyringProvider, setup_py: SetupPyStrategy, connectivity: Connectivity, @@ -144,6 +145,7 @@ pub(crate) async fn pip_sync( .native_tls(native_tls) .connectivity(connectivity) .index_urls(index_locations.index_urls()) + .index_strategy(index_strategy) .keyring_provider(keyring_provider) .markers(venv.interpreter().markers()) .platform(venv.interpreter().platform()) diff --git a/crates/uv/src/commands/venv.rs b/crates/uv/src/commands/venv.rs index 6faa5e131..e6c842cbd 100644 --- a/crates/uv/src/commands/venv.rs +++ b/crates/uv/src/commands/venv.rs @@ -21,7 +21,8 @@ use uv_fs::Simplified; use uv_interpreter::{find_default_python, find_requested_python, Error}; use uv_resolver::{InMemoryIndex, OptionsBuilder}; use uv_types::{ - BuildContext, BuildIsolation, ConfigSettings, InFlight, NoBinary, NoBuild, SetupPyStrategy, + BuildContext, BuildIsolation, ConfigSettings, InFlight, IndexStrategy, NoBinary, NoBuild, + SetupPyStrategy, }; use crate::commands::ExitStatus; @@ -34,6 +35,7 @@ pub(crate) async fn venv( path: &Path, python_request: Option<&str>, index_locations: &IndexLocations, + index_strategy: IndexStrategy, keyring_provider: KeyringProvider, prompt: uv_virtualenv::Prompt, system_site_packages: bool, @@ -48,6 +50,7 @@ pub(crate) async fn venv( path, python_request, index_locations, + index_strategy, keyring_provider, prompt, system_site_packages, @@ -93,6 +96,7 @@ async fn venv_impl( path: &Path, python_request: Option<&str>, index_locations: &IndexLocations, + index_strategy: IndexStrategy, keyring_provider: KeyringProvider, prompt: uv_virtualenv::Prompt, system_site_packages: bool, @@ -150,6 +154,7 @@ async fn venv_impl( let client = RegistryClientBuilder::new(cache.clone()) .native_tls(native_tls) .index_urls(index_locations.index_urls()) + .index_strategy(index_strategy) .keyring_provider(keyring_provider) .connectivity(connectivity) .markers(interpreter.markers()) diff --git a/crates/uv/src/logging.rs b/crates/uv/src/logging.rs index 67c491a4d..6487b0fb6 100644 --- a/crates/uv/src/logging.rs +++ b/crates/uv/src/logging.rs @@ -124,7 +124,7 @@ pub(crate) fn setup_logging( } Level::Verbose | Level::ExtraVerbose => { // Show `DEBUG` messages from the CLI crate, but allow `RUST_LOG` to override. - Directive::from_str("uv=debug").unwrap() + Directive::from_str("uv=trace").unwrap() } }; diff --git a/crates/uv/src/main.rs b/crates/uv/src/main.rs index ab8d4d71f..913069bf4 100644 --- a/crates/uv/src/main.rs +++ b/crates/uv/src/main.rs @@ -20,11 +20,11 @@ use uv_interpreter::PythonVersion; use uv_normalize::{ExtraName, PackageName}; use uv_requirements::{ExtrasSpecification, RequirementsSource}; use uv_resolver::{AnnotationStyle, DependencyMode, PreReleaseMode, ResolutionMode}; -use uv_types::NoBinary; use uv_types::{ ConfigSettingEntry, ConfigSettings, NoBuild, PackageNameSpecifier, Reinstall, SetupPyStrategy, Upgrade, }; +use uv_types::{IndexStrategy, NoBinary}; use crate::commands::{extra_name_with_clap_error, ExitStatus, ListFormat, VersionFormat}; use crate::compat::CompatArgs; @@ -384,6 +384,15 @@ struct PipCompileArgs { #[clap(long, conflicts_with = "index_url", conflicts_with = "extra_index_url")] no_index: bool, + /// The strategy to use when resolving against multiple index URLs. + /// + /// By default, `uv` will stop at the first index on which a given package is available, and + /// limit resolutions to those present on that first index. This prevents "dependency confusion" + /// attacks, whereby an attack can upload a malicious package under the same name to a secondary + /// index. + #[clap(long, default_value_t, value_enum, env = "UV_INDEX_STRATEGY")] + index_strategy: IndexStrategy, + /// Attempt to use `keyring` for authentication for index urls /// /// Due to not having Python imports, only `--keyring-provider subprocess` argument is currently @@ -570,6 +579,15 @@ struct PipSyncArgs { #[clap(long, conflicts_with = "index_url", conflicts_with = "extra_index_url")] no_index: bool, + /// The strategy to use when resolving against multiple index URLs. + /// + /// By default, `uv` will stop at the first index on which a given package is available, and + /// limit resolutions to those present on that first index. This prevents "dependency confusion" + /// attacks, whereby an attack can upload a malicious package under the same name to a secondary + /// index. + #[clap(long, default_value_t, value_enum, env = "UV_INDEX_STRATEGY")] + index_strategy: IndexStrategy, + /// Attempt to use `keyring` for authentication for index urls /// /// Function's similar to `pip`'s `--keyring-provider subprocess` argument, @@ -835,6 +853,15 @@ struct PipInstallArgs { #[clap(long, conflicts_with = "index_url", conflicts_with = "extra_index_url")] no_index: bool, + /// The strategy to use when resolving against multiple index URLs. + /// + /// By default, `uv` will stop at the first index on which a given package is available, and + /// limit resolutions to those present on that first index. This prevents "dependency confusion" + /// attacks, whereby an attack can upload a malicious package under the same name to a secondary + /// index. + #[clap(long, default_value_t, value_enum, env = "UV_INDEX_STRATEGY")] + index_strategy: IndexStrategy, + /// Attempt to use `keyring` for authentication for index urls /// /// Due to not having Python imports, only `--keyring-provider subprocess` argument is currently @@ -1336,6 +1363,15 @@ struct VenvArgs { #[clap(long, conflicts_with = "index_url", conflicts_with = "extra_index_url")] no_index: bool, + /// The strategy to use when resolving against multiple index URLs. + /// + /// By default, `uv` will stop at the first index on which a given package is available, and + /// limit resolutions to those present on that first index. This prevents "dependency confusion" + /// attacks, whereby an attack can upload a malicious package under the same name to a secondary + /// index. + #[clap(long, default_value_t, value_enum, env = "UV_INDEX_STRATEGY")] + index_strategy: IndexStrategy, + /// Attempt to use `keyring` for authentication for index urls /// /// Due to not having Python imports, only `--keyring-provider subprocess` argument is currently @@ -1552,6 +1588,7 @@ async fn run() -> Result { args.emit_find_links, args.emit_marker_expression, index_urls, + args.index_strategy, args.keyring_provider, setup_py, config_settings, @@ -1608,6 +1645,7 @@ async fn run() -> Result { args.link_mode, args.compile, index_urls, + args.index_strategy, args.keyring_provider, setup_py, if args.offline { @@ -1701,6 +1739,7 @@ async fn run() -> Result { dependency_mode, upgrade, index_urls, + args.index_strategy, args.keyring_provider, reinstall, args.link_mode, @@ -1833,6 +1872,7 @@ async fn run() -> Result { &args.name, args.python.as_deref(), &index_locations, + args.index_strategy, args.keyring_provider, uv_virtualenv::Prompt::from_args(prompt), args.system_site_packages, diff --git a/crates/uv/tests/common/mod.rs b/crates/uv/tests/common/mod.rs index f9f384165..4d09b7ac7 100644 --- a/crates/uv/tests/common/mod.rs +++ b/crates/uv/tests/common/mod.rs @@ -139,13 +139,23 @@ impl TestContext { /// * Set a cutoff for versions used in the resolution so the snapshots don't change after a new release. /// * Set the venv to a fresh `.venv` in `temp_dir`. pub fn compile(&self) -> std::process::Command { + let mut command = self.compile_without_exclude_newer(); + command.arg("--exclude-newer").arg(EXCLUDE_NEWER); + command + } + + /// Create a `pip compile` command with no `--exclude-newer` option. + /// + /// One should avoid using this in tests to the extent possible because + /// it can result in tests failing when the index state changes. Therefore, + /// if you use this, there should be some other kind of mitigation in place. + /// For example, pinning package versions. + pub fn compile_without_exclude_newer(&self) -> std::process::Command { let mut cmd = std::process::Command::new(get_bin()); cmd.arg("pip") .arg("compile") .arg("--cache-dir") .arg(self.cache_dir.path()) - .arg("--exclude-newer") - .arg(EXCLUDE_NEWER) .env("VIRTUAL_ENV", self.venv.as_os_str()) .current_dir(self.temp_dir.path()); diff --git a/crates/uv/tests/pip_compile.rs b/crates/uv/tests/pip_compile.rs index 2debbb05b..4cca0a55d 100644 --- a/crates/uv/tests/pip_compile.rs +++ b/crates/uv/tests/pip_compile.rs @@ -7040,3 +7040,110 @@ requires-python = ">3.8" Ok(()) } + +/// Install a package via `--extra-index-url`. +/// +/// If the package exists exist on the "extra" index, but at an incompatible version, the +/// resolution should fail by default (even though a compatible version exists on the "primary" +/// index). +#[test] +fn compile_index_url_first_match() -> Result<()> { + let context = TestContext::new("3.12"); + + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str("jinja2==3.1.0")?; + + uv_snapshot!(context.compile() + .arg("--index-url") + .arg("https://pypi.org/simple") + .arg("--extra-index-url") + .arg("https://download.pytorch.org/whl/cpu") + .arg("requirements.in") + .arg("--no-deps"), @r###" + success: false + exit_code: 1 + ----- stdout ----- + + ----- stderr ----- + × No solution found when resolving dependencies: + ╰─▶ Because there is no version of jinja2==3.1.0 and you require + jinja2==3.1.0, we can conclude that the requirements are unsatisfiable. + "### + ); + + Ok(()) +} + +/// Install a package via `--extra-index-url`. +/// +/// If the package exists exist on the "extra" index, but at an incompatible version, the +/// resolution should fallback to the "primary" index when `--index-strategy unsafe-any-match` +/// is provided. +#[test] +fn compile_index_url_fallback() -> Result<()> { + let context = TestContext::new("3.12"); + + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str("jinja2==3.1.0")?; + + uv_snapshot!(context.compile() + .arg("--index-strategy") + .arg("unsafe-any-match") + .arg("--index-url") + .arg("https://pypi.org/simple") + .arg("--extra-index-url") + .arg("https://download.pytorch.org/whl/cpu") + .arg("requirements.in") + .arg("--no-deps"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] --exclude-newer 2024-03-25T00:00:00Z --index-strategy unsafe-any-match requirements.in --no-deps + jinja2==3.1.0 + + ----- stderr ----- + Resolved 1 package in [TIME] + "### + ); + + Ok(()) +} + +/// Install a package via `--extra-index-url`. +/// +/// If the package exists exist on the "extra" index at a compatible version, the resolver should +/// prefer it, even if a newer versions exists on the "primary" index. +/// +/// In this case, Jinja 3.1.2 is hosted on the "extra" index, but newer versions are available on +/// the "primary" index. +#[test] +fn compile_index_url_fallback_prefer_primary() -> Result<()> { + let context = TestContext::new("3.12"); + + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str("jinja2")?; + + uv_snapshot!(context.compile_without_exclude_newer() + .arg("--index-strategy") + .arg("unsafe-any-match") + .arg("--index-url") + .arg("https://pypi.org/simple") + .arg("--extra-index-url") + .arg("https://download.pytorch.org/whl/cpu") + .arg("requirements.in") + .arg("--no-deps"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] --index-strategy unsafe-any-match requirements.in --no-deps + jinja2==3.1.2 + + ----- stderr ----- + Resolved 1 package in [TIME] + "### + ); + + Ok(()) +}