Add wheel variant support

This commit is contained in:
konstin 2025-07-21 18:50:24 +02:00
parent e4d193a5f8
commit cc81ee9fcd
142 changed files with 9239 additions and 2334 deletions

59
Cargo.lock generated
View File

@ -4181,6 +4181,7 @@ version = "1.0.145"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
dependencies = [ dependencies = [
"indexmap",
"itoa", "itoa",
"memchr", "memchr",
"ryu", "ryu",
@ -5492,6 +5493,7 @@ dependencies = [
"uv-torch", "uv-torch",
"uv-trampoline-builder", "uv-trampoline-builder",
"uv-types", "uv-types",
"uv-variants",
"uv-version", "uv-version",
"uv-virtualenv", "uv-virtualenv",
"uv-warnings", "uv-warnings",
@ -5831,6 +5833,7 @@ dependencies = [
"uv-small-str", "uv-small-str",
"uv-static", "uv-static",
"uv-torch", "uv-torch",
"uv-variants",
"uv-version", "uv-version",
"uv-warnings", "uv-warnings",
"wiremock", "wiremock",
@ -5961,6 +5964,8 @@ dependencies = [
"uv-python", "uv-python",
"uv-resolver", "uv-resolver",
"uv-types", "uv-types",
"uv-variant-frontend",
"uv-variants",
"uv-version", "uv-version",
"uv-workspace", "uv-workspace",
] ]
@ -5976,6 +5981,7 @@ dependencies = [
"futures", "futures",
"indoc", "indoc",
"insta", "insta",
"itertools 0.14.0",
"nanoid", "nanoid",
"owo-colors", "owo-colors",
"reqwest", "reqwest",
@ -6002,12 +6008,14 @@ dependencies = [
"uv-git-types", "uv-git-types",
"uv-metadata", "uv-metadata",
"uv-normalize", "uv-normalize",
"uv-once-map",
"uv-pep440", "uv-pep440",
"uv-pep508", "uv-pep508",
"uv-platform-tags", "uv-platform-tags",
"uv-pypi-types", "uv-pypi-types",
"uv-redacted", "uv-redacted",
"uv-types", "uv-types",
"uv-variants",
"uv-workspace", "uv-workspace",
"walkdir", "walkdir",
"zip", "zip",
@ -6067,6 +6075,7 @@ dependencies = [
"uv-pypi-types", "uv-pypi-types",
"uv-redacted", "uv-redacted",
"uv-small-str", "uv-small-str",
"uv-variants",
"uv-warnings", "uv-warnings",
] ]
@ -6482,6 +6491,7 @@ dependencies = [
"anyhow", "anyhow",
"hashbrown 0.16.1", "hashbrown 0.16.1",
"indexmap", "indexmap",
"indoc",
"insta", "insta",
"itertools 0.14.0", "itertools 0.14.0",
"jiff", "jiff",
@ -6708,6 +6718,7 @@ dependencies = [
"uv-static", "uv-static",
"uv-torch", "uv-torch",
"uv-types", "uv-types",
"uv-variants",
"uv-version", "uv-version",
"uv-warnings", "uv-warnings",
"uv-workspace", "uv-workspace",
@ -6897,12 +6908,60 @@ dependencies = [
"uv-normalize", "uv-normalize",
"uv-once-map", "uv-once-map",
"uv-pep440", "uv-pep440",
"uv-pep508",
"uv-pypi-types", "uv-pypi-types",
"uv-python", "uv-python",
"uv-redacted", "uv-redacted",
"uv-variants",
"uv-workspace", "uv-workspace",
] ]
[[package]]
name = "uv-variant-frontend"
version = "0.0.3"
dependencies = [
"anstream",
"anyhow",
"fs-err",
"indoc",
"owo-colors",
"rustc-hash",
"serde_json",
"tempfile",
"thiserror 2.0.17",
"tokio",
"tracing",
"uv-configuration",
"uv-distribution-types",
"uv-fs",
"uv-preview",
"uv-python",
"uv-static",
"uv-types",
"uv-variants",
"uv-virtualenv",
]
[[package]]
name = "uv-variants"
version = "0.0.3"
dependencies = [
"indexmap",
"insta",
"itertools 0.14.0",
"rustc-hash",
"serde",
"serde_json",
"thiserror 2.0.17",
"tracing",
"uv-distribution-filename",
"uv-normalize",
"uv-once-map",
"uv-pep440",
"uv-pep508",
"uv-pypi-types",
]
[[package]] [[package]]
name = "uv-version" name = "uv-version"
version = "0.9.14" version = "0.9.14"

View File

@ -70,6 +70,8 @@ uv-tool = { version = "0.0.4", path = "crates/uv-tool" }
uv-torch = { version = "0.0.4", path = "crates/uv-torch" } uv-torch = { version = "0.0.4", path = "crates/uv-torch" }
uv-trampoline-builder = { version = "0.0.4", path = "crates/uv-trampoline-builder" } uv-trampoline-builder = { version = "0.0.4", path = "crates/uv-trampoline-builder" }
uv-types = { version = "0.0.4", path = "crates/uv-types" } uv-types = { version = "0.0.4", path = "crates/uv-types" }
uv-variant-frontend = { path = "crates/uv-variant-frontend" }
uv-variants = { path = "crates/uv-variants" }
uv-version = { version = "0.9.14", path = "crates/uv-version" } uv-version = { version = "0.9.14", path = "crates/uv-version" }
uv-virtualenv = { version = "0.0.4", path = "crates/uv-virtualenv" } uv-virtualenv = { version = "0.0.4", path = "crates/uv-virtualenv" }
uv-warnings = { version = "0.0.4", path = "crates/uv-warnings" } uv-warnings = { version = "0.0.4", path = "crates/uv-warnings" }
@ -166,7 +168,7 @@ security-framework = { version = "3" }
self-replace = { version = "1.5.0" } self-replace = { version = "1.5.0" }
serde = { version = "1.0.210", features = ["derive", "rc"] } serde = { version = "1.0.210", features = ["derive", "rc"] }
serde-untagged = { version = "0.1.6" } serde-untagged = { version = "0.1.6" }
serde_json = { version = "1.0.128" } serde_json = { version = "1.0.128", features = ["preserve_order"] }
sha2 = { version = "0.10.8" } sha2 = { version = "0.10.8" }
smallvec = { version = "1.13.2" } smallvec = { version = "1.13.2" }
spdx = { version = "0.12.0" } spdx = { version = "0.12.0" }

View File

@ -597,7 +597,7 @@ mod tests {
// Check that the source dist is reproducible across platforms. // Check that the source dist is reproducible across platforms.
assert_snapshot!( assert_snapshot!(
format!("{:x}", sha2::Sha256::digest(fs_err::read(&source_dist_path).unwrap())), format!("{:x}", sha2::Sha256::digest(fs_err::read(&source_dist_path).unwrap())),
@"871d1f859140721b67cbeaca074e7a2740c88c38028d0509eba87d1285f1da9e" @"590388c63ef4379eef57bedafffc6522dd2e3b84e689fe55ba3b1e7f2de8cc13"
); );
// Check both the files we report and the actual files // Check both the files we report and the actual files
assert_snapshot!(format_file_list(build.source_dist_list_files, src.path()), @r" assert_snapshot!(format_file_list(build.source_dist_list_files, src.path()), @r"

View File

@ -34,6 +34,7 @@ uv-small-str = { workspace = true }
uv-redacted = { workspace = true } uv-redacted = { workspace = true }
uv-static = { workspace = true } uv-static = { workspace = true }
uv-torch = { workspace = true } uv-torch = { workspace = true }
uv-variants = { workspace = true }
uv-version = { workspace = true } uv-version = { workspace = true }
uv-warnings = { workspace = true } uv-warnings = { workspace = true }

View File

@ -6,6 +6,7 @@ use std::ops::Deref;
use std::path::PathBuf; use std::path::PathBuf;
use uv_distribution_filename::{WheelFilename, WheelFilenameError}; use uv_distribution_filename::{WheelFilename, WheelFilenameError};
use uv_distribution_types::VariantsJsonFilename;
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_redacted::DisplaySafeUrl; use uv_redacted::DisplaySafeUrl;
@ -278,6 +279,10 @@ pub enum ErrorKind {
#[error("Package `{0}` was not found in the local index")] #[error("Package `{0}` was not found in the local index")]
LocalPackageNotFound(PackageName), LocalPackageNotFound(PackageName),
/// The `variants.json` file was not found in the local (file-based) index.
#[error("Variants JSON file `{0}` was not found in the local index")]
VariantsJsonNotFile(VariantsJsonFilename),
/// The root was not found in the local (file-based) index. /// The root was not found in the local (file-based) index.
#[error("Local index not found at: `{}`", _0.display())] #[error("Local index not found at: `{}`", _0.display())]
LocalIndexNotFound(PathBuf), LocalIndexNotFound(PathBuf),
@ -368,6 +373,9 @@ pub enum ErrorKind {
#[error("Invalid cache control header: `{0}`")] #[error("Invalid cache control header: `{0}`")]
InvalidCacheControl(String), InvalidCacheControl(String),
#[error("Invalid variants.json format: {0}")]
VariantsJsonFormat(DisplaySafeUrl, #[source] serde_json::Error),
} }
impl ErrorKind { impl ErrorKind {

View File

@ -7,8 +7,7 @@ use url::Url;
use uv_cache::{Cache, CacheBucket}; use uv_cache::{Cache, CacheBucket};
use uv_cache_key::cache_digest; use uv_cache_key::cache_digest;
use uv_distribution_filename::DistFilename; use uv_distribution_types::{File, FileLocation, IndexEntryFilename, IndexUrl, UrlString};
use uv_distribution_types::{File, FileLocation, IndexUrl, UrlString};
use uv_pypi_types::HashDigests; use uv_pypi_types::HashDigests;
use uv_redacted::DisplaySafeUrl; use uv_redacted::DisplaySafeUrl;
use uv_small_str::SmallString; use uv_small_str::SmallString;
@ -40,7 +39,7 @@ pub enum FindLinksDirectoryError {
/// An entry in a `--find-links` index. /// An entry in a `--find-links` index.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct FlatIndexEntry { pub struct FlatIndexEntry {
pub filename: DistFilename, pub filename: IndexEntryFilename,
pub file: File, pub file: File,
pub index: IndexUrl, pub index: IndexUrl,
} }
@ -238,7 +237,9 @@ impl<'a> FlatIndexClient<'a> {
}) })
.filter_map(|file| { .filter_map(|file| {
Some(FlatIndexEntry { Some(FlatIndexEntry {
filename: DistFilename::try_from_normalized_filename(&file.filename)?, filename: IndexEntryFilename::try_from_normalized_filename(
&file.filename,
)?,
file, file,
index: flat_index.clone(), index: flat_index.clone(),
}) })
@ -308,9 +309,10 @@ impl<'a> FlatIndexClient<'a> {
zstd: None, zstd: None,
}; };
let Some(filename) = DistFilename::try_from_normalized_filename(filename) else { // Try to parse as a distribution filename first
let Some(filename) = IndexEntryFilename::try_from_normalized_filename(filename) else {
debug!( debug!(
"Ignoring `--find-links` entry (expected a wheel or source distribution filename): {}", "Ignoring `--find-links` entry (expected a wheel, source distribution, or variants.json filename): {}",
entry.path().display() entry.path().display()
); );
continue; continue;
@ -338,6 +340,7 @@ mod tests {
use fs_err::File; use fs_err::File;
use std::io::Write; use std::io::Write;
use tempfile::tempdir; use tempfile::tempdir;
use uv_distribution_filename::DistFilename;
#[test] #[test]
fn read_from_directory_sorts_distributions() { fn read_from_directory_sorts_distributions() {

View File

@ -272,6 +272,11 @@ impl SimpleIndexHtml {
return None; return None;
} }
//
if project_name.ends_with("-variants.json") {
return None;
}
PackageName::from_str(project_name).ok() PackageName::from_str(project_name).ok()
} }
} }
@ -1603,4 +1608,63 @@ mod tests {
} }
"#); "#);
} }
#[test]
fn parse_variants_json() {
// A variants.json without wheels doesn't make much sense, but it's sufficient to test
// parsing.
let text = r#"
<!DOCTYPE html>
<html>
<body>
<h1>Links for jinja2</h1>
<a href="/whl/jinja2-3.1.2-variants.json#sha256=6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61">jinja2-3.1.2-variants.json</a><br/>
</body>
</html>
<!--TIMESTAMP 1703347410-->
"#;
let base = DisplaySafeUrl::parse("https://download.pytorch.org/whl/jinja2/").unwrap();
let result = SimpleDetailHTML::parse(text, &base).unwrap();
insta::assert_debug_snapshot!(result, @r#"
SimpleDetailHTML {
base: BaseUrl(
DisplaySafeUrl {
scheme: "https",
cannot_be_a_base: false,
username: "",
password: None,
host: Some(
Domain(
"download.pytorch.org",
),
),
port: None,
path: "/whl/jinja2/",
query: None,
fragment: None,
},
),
files: [
PypiFile {
core_metadata: None,
filename: "jinja2-3.1.2-variants.json",
hashes: Hashes {
md5: None,
sha256: Some(
"6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61",
),
sha384: None,
sha512: None,
blake2b: None,
},
requires_python: None,
size: None,
upload_time: None,
url: "/whl/jinja2-3.1.2-variants.json",
yanked: None,
},
],
}
"#);
}
} }

View File

@ -21,8 +21,9 @@ use uv_configuration::IndexStrategy;
use uv_configuration::KeyringProviderType; use uv_configuration::KeyringProviderType;
use uv_distribution_filename::{DistFilename, SourceDistFilename, WheelFilename}; use uv_distribution_filename::{DistFilename, SourceDistFilename, WheelFilename};
use uv_distribution_types::{ use uv_distribution_types::{
BuiltDist, File, IndexCapabilities, IndexFormat, IndexLocations, IndexMetadataRef, BuiltDist, File, IndexCapabilities, IndexEntryFilename, IndexFormat, IndexLocations,
IndexStatusCodeDecision, IndexStatusCodeStrategy, IndexUrl, IndexUrls, Name, IndexMetadataRef, IndexStatusCodeDecision, IndexStatusCodeStrategy, IndexUrl, IndexUrls, Name,
RegistryVariantsJson, VariantsJsonFilename,
}; };
use uv_metadata::{read_metadata_async_seek, read_metadata_async_stream}; use uv_metadata::{read_metadata_async_seek, read_metadata_async_stream};
use uv_normalize::PackageName; use uv_normalize::PackageName;
@ -35,6 +36,7 @@ use uv_pypi_types::{
use uv_redacted::DisplaySafeUrl; use uv_redacted::DisplaySafeUrl;
use uv_small_str::SmallString; use uv_small_str::SmallString;
use uv_torch::TorchStrategy; use uv_torch::TorchStrategy;
use uv_variants::variants_json::VariantsJsonContent;
use crate::base_client::{BaseClientBuilder, ExtraMiddleware, RedirectPolicy}; use crate::base_client::{BaseClientBuilder, ExtraMiddleware, RedirectPolicy};
use crate::cached_client::CacheControl; use crate::cached_client::CacheControl;
@ -867,6 +869,85 @@ impl RegistryClient {
OwnedArchive::from_unarchived(&metadata) OwnedArchive::from_unarchived(&metadata)
} }
/// Fetch the variants.json contents from a remote index (cached) a local index.
pub async fn fetch_variants_json(
&self,
variants_json: &RegistryVariantsJson,
) -> Result<VariantsJsonContent, Error> {
let url = variants_json
.file
.url
.to_url()
.map_err(ErrorKind::InvalidUrl)?;
// If the URL is a file URL, load the variants directly from the file system.
let variants_json = if url.scheme() == "file" {
let path = url
.to_file_path()
.map_err(|()| ErrorKind::NonFileUrl(url.clone()))?;
let bytes = match fs_err::tokio::read(&path).await {
Ok(text) => text,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
return Err(Error::from(ErrorKind::VariantsJsonNotFile(
variants_json.filename.clone(),
)));
}
Err(err) => {
return Err(Error::from(ErrorKind::Io(err)));
}
};
info_span!("parse_variants_json")
.in_scope(|| serde_json::from_slice::<VariantsJsonContent>(&bytes))
.map_err(|err| ErrorKind::VariantsJsonFormat(url, err))?
} else {
let cache_entry = self.cache.entry(
CacheBucket::Wheels,
WheelCache::Index(&variants_json.index)
.wheel_dir(variants_json.filename.name.as_ref()),
format!("variants-{}.msgpack", variants_json.filename.cache_key()),
);
let cache_control = match self.connectivity {
Connectivity::Online => {
if let Some(header) = self
.index_urls
.artifact_cache_control_for(&variants_json.index)
{
CacheControl::Override(header)
} else {
CacheControl::from(
self.cache
.freshness(&cache_entry, Some(&variants_json.filename.name), None)
.map_err(ErrorKind::Io)?,
)
}
}
Connectivity::Offline => CacheControl::AllowStale,
};
let response_callback = async |response: Response| {
let bytes = response
.bytes()
.await
.map_err(|err| ErrorKind::from_reqwest(url.clone(), err))?;
info_span!("parse_variants_json")
.in_scope(|| serde_json::from_slice::<VariantsJsonContent>(&bytes))
.map_err(|err| Error::from(ErrorKind::VariantsJsonFormat(url.clone(), err)))
};
let req = self
.uncached_client(&url)
.get(Url::from(url.clone()))
.build()
.map_err(|err| ErrorKind::from_reqwest(url.clone(), err))?;
self.cached_client()
.get_serde_with_retry(req, &cache_entry, cache_control, response_callback)
.await?
};
Ok(variants_json)
}
/// Fetch the metadata for a remote wheel file. /// Fetch the metadata for a remote wheel file.
/// ///
/// For a remote wheel, we try the following ways to fetch the metadata: /// For a remote wheel, we try the following ways to fetch the metadata:
@ -1263,19 +1344,28 @@ impl FlatIndexCache {
pub struct VersionFiles { pub struct VersionFiles {
pub wheels: Vec<VersionWheel>, pub wheels: Vec<VersionWheel>,
pub source_dists: Vec<VersionSourceDist>, pub source_dists: Vec<VersionSourceDist>,
pub variant_jsons: Vec<VersionVariantJson>,
} }
impl VersionFiles { impl VersionFiles {
fn push(&mut self, filename: DistFilename, file: File) { fn push(&mut self, filename: IndexEntryFilename, file: File) {
match filename { match filename {
DistFilename::WheelFilename(name) => self.wheels.push(VersionWheel { name, file }), IndexEntryFilename::DistFilename(DistFilename::WheelFilename(name)) => {
DistFilename::SourceDistFilename(name) => { self.wheels.push(VersionWheel { name, file });
}
IndexEntryFilename::DistFilename(DistFilename::SourceDistFilename(name)) => {
self.source_dists.push(VersionSourceDist { name, file }); self.source_dists.push(VersionSourceDist { name, file });
} }
IndexEntryFilename::VariantJson(variants_json) => {
self.variant_jsons.push(VersionVariantJson {
name: variants_json,
file,
});
}
} }
} }
pub fn all(self) -> impl Iterator<Item = (DistFilename, File)> { pub fn dists(self) -> impl Iterator<Item = (DistFilename, File)> {
self.source_dists self.source_dists
.into_iter() .into_iter()
.map(|VersionSourceDist { name, file }| (DistFilename::SourceDistFilename(name), file)) .map(|VersionSourceDist { name, file }| (DistFilename::SourceDistFilename(name), file))
@ -1285,6 +1375,30 @@ impl VersionFiles {
.map(|VersionWheel { name, file }| (DistFilename::WheelFilename(name), file)), .map(|VersionWheel { name, file }| (DistFilename::WheelFilename(name), file)),
) )
} }
pub fn all(self) -> impl Iterator<Item = (IndexEntryFilename, File)> {
self.source_dists
.into_iter()
.map(|VersionSourceDist { name, file }| {
(
IndexEntryFilename::DistFilename(DistFilename::SourceDistFilename(name)),
file,
)
})
.chain(self.wheels.into_iter().map(|VersionWheel { name, file }| {
(
IndexEntryFilename::DistFilename(DistFilename::WheelFilename(name)),
file,
)
}))
.chain(
self.variant_jsons
.into_iter()
.map(|VersionVariantJson { name, file }| {
(IndexEntryFilename::VariantJson(name), file)
}),
)
}
} }
#[derive(Debug, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] #[derive(Debug, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)]
@ -1301,6 +1415,13 @@ pub struct VersionSourceDist {
pub file: File, pub file: File,
} }
#[derive(Debug, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)]
#[rkyv(derive(Debug))]
pub struct VersionVariantJson {
pub name: VariantsJsonFilename,
pub file: File,
}
/// The list of projects available in a Simple API index. /// The list of projects available in a Simple API index.
#[derive(Default, Debug, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] #[derive(Default, Debug, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)]
#[rkyv(derive(Debug))] #[rkyv(derive(Debug))]
@ -1372,7 +1493,8 @@ impl SimpleDetailMetadata {
// Group the distributions by version and kind // Group the distributions by version and kind
for file in files { for file in files {
let Some(filename) = DistFilename::try_from_filename(&file.filename, package_name) let Some(filename) =
IndexEntryFilename::try_from_filename(&file.filename, package_name)
else { else {
warn!("Skipping file for {package_name}: {}", file.filename); warn!("Skipping file for {package_name}: {}", file.filename);
continue; continue;
@ -1430,7 +1552,8 @@ impl SimpleDetailMetadata {
continue; continue;
} }
}; };
let Some(filename) = DistFilename::try_from_filename(&file.filename, package_name) let Some(filename) =
IndexEntryFilename::try_from_filename(&file.filename, package_name)
else { else {
warn!("Skipping file for {package_name}: {}", file.filename); warn!("Skipping file for {package_name}: {}", file.filename);
continue; continue;

View File

@ -1,6 +1,7 @@
#[cfg(feature = "schemars")] #[cfg(feature = "schemars")]
use std::borrow::Cow; use std::borrow::Cow;
use std::{fmt::Formatter, str::FromStr}; use std::fmt::Formatter;
use std::str::FromStr;
use uv_pep440::{Version, VersionSpecifier, VersionSpecifiers, VersionSpecifiersParseError}; use uv_pep440::{Version, VersionSpecifier, VersionSpecifiers, VersionSpecifiersParseError};

View File

@ -33,6 +33,8 @@ uv-pypi-types = { workspace = true }
uv-python = { workspace = true } uv-python = { workspace = true }
uv-resolver = { workspace = true } uv-resolver = { workspace = true }
uv-types = { workspace = true } uv-types = { workspace = true }
uv-variant-frontend = { workspace = true }
uv-variants = { workspace = true }
uv-version = { workspace = true } uv-version = { workspace = true }
uv-workspace = { workspace = true } uv-workspace = { workspace = true }

View File

@ -40,6 +40,9 @@ use uv_types::{
AnyErrorBuild, BuildArena, BuildContext, BuildIsolation, BuildStack, EmptyInstalledPackages, AnyErrorBuild, BuildArena, BuildContext, BuildIsolation, BuildStack, EmptyInstalledPackages,
HashStrategy, InFlight, HashStrategy, InFlight,
}; };
use uv_variant_frontend::VariantBuild;
use uv_variants::cache::VariantProviderCache;
use uv_variants::variants_json::Provider;
use uv_workspace::WorkspaceCache; use uv_workspace::WorkspaceCache;
#[derive(Debug, Error)] #[derive(Debug, Error)]
@ -177,6 +180,7 @@ impl<'a> BuildDispatch<'a> {
#[allow(refining_impl_trait)] #[allow(refining_impl_trait)]
impl BuildContext for BuildDispatch<'_> { impl BuildContext for BuildDispatch<'_> {
type SourceDistBuilder = SourceBuild; type SourceDistBuilder = SourceBuild;
type VariantsBuilder = VariantBuild;
async fn interpreter(&self) -> &Interpreter { async fn interpreter(&self) -> &Interpreter {
self.interpreter self.interpreter
@ -190,6 +194,10 @@ impl BuildContext for BuildDispatch<'_> {
&self.shared_state.git &self.shared_state.git
} }
fn variants(&self) -> &VariantProviderCache {
self.shared_state.index.variant_providers()
}
fn build_arena(&self) -> &BuildArena<SourceBuild> { fn build_arena(&self) -> &BuildArena<SourceBuild> {
&self.shared_state.build_arena &self.shared_state.build_arena
} }
@ -559,6 +567,26 @@ impl BuildContext for BuildDispatch<'_> {
Ok(Some(filename)) Ok(Some(filename))
} }
async fn setup_variants<'data>(
&'data self,
backend_name: String,
backend: &'data Provider,
build_output: BuildOutput,
) -> anyhow::Result<VariantBuild> {
let builder = VariantBuild::setup(
backend_name,
backend,
self.interpreter,
self,
self.build_extra_env_vars.clone(),
build_output,
self.concurrency.builds,
)
.boxed_local()
.await?;
Ok(builder)
}
} }
/// Shared state used during resolution and installation. /// Shared state used during resolution and installation.

View File

@ -10,6 +10,7 @@ use uv_platform_tags::{
use crate::splitter::MemchrSplitter; use crate::splitter::MemchrSplitter;
use crate::wheel_tag::{WheelTag, WheelTagLarge, WheelTagSmall}; use crate::wheel_tag::{WheelTag, WheelTagLarge, WheelTagSmall};
use crate::{InvalidVariantLabel, VariantLabel};
/// The expanded wheel tags as stored in a `WHEEL` file. /// The expanded wheel tags as stored in a `WHEEL` file.
/// ///
@ -81,6 +82,8 @@ pub enum ExpandedTagError {
InvalidAbiTag(String, #[source] ParseAbiTagError), InvalidAbiTag(String, #[source] ParseAbiTagError),
#[error("The wheel tag \"{0}\" contains an invalid platform tag")] #[error("The wheel tag \"{0}\" contains an invalid platform tag")]
InvalidPlatformTag(String, #[source] ParsePlatformTagError), InvalidPlatformTag(String, #[source] ParsePlatformTagError),
#[error("The wheel tag \"{0}\" contains an invalid variant label")]
InvalidVariantLabel(String, #[source] InvalidVariantLabel),
} }
/// Parse an expanded (i.e., simplified) wheel tag, e.g. `py3-none-any`. /// Parse an expanded (i.e., simplified) wheel tag, e.g. `py3-none-any`.
@ -100,13 +103,15 @@ fn parse_expanded_tag(tag: &str) -> Result<WheelTag, ExpandedTagError> {
let Some(abi_tag_index) = splitter.next() else { let Some(abi_tag_index) = splitter.next() else {
return Err(ExpandedTagError::MissingPlatformTag(tag.to_string())); return Err(ExpandedTagError::MissingPlatformTag(tag.to_string()));
}; };
let variant = splitter.next();
if splitter.next().is_some() { if splitter.next().is_some() {
return Err(ExpandedTagError::ExtraSegment(tag.to_string())); return Err(ExpandedTagError::ExtraSegment(tag.to_string()));
} }
let python_tag = &tag[..python_tag_index]; let python_tag = &tag[..python_tag_index];
let abi_tag = &tag[python_tag_index + 1..abi_tag_index]; let abi_tag = &tag[python_tag_index + 1..abi_tag_index];
let platform_tag = &tag[abi_tag_index + 1..]; let platform_tag = &tag[abi_tag_index + 1..variant.unwrap_or(tag.len())];
let variant = variant.map(|variant| &tag[variant + 1..]);
let is_small = memchr(b'.', tag.as_bytes()).is_none(); let is_small = memchr(b'.', tag.as_bytes()).is_none();
@ -137,6 +142,10 @@ fn parse_expanded_tag(tag: &str) -> Result<WheelTag, ExpandedTagError> {
.map(PlatformTag::from_str) .map(PlatformTag::from_str)
.filter_map(Result::ok) .filter_map(Result::ok)
.collect(), .collect(),
variant: variant
.map(VariantLabel::from_str)
.transpose()
.map_err(|err| ExpandedTagError::InvalidVariantLabel(tag.to_string(), err))?,
repr: tag.into(), repr: tag.into(),
}), }),
}) })
@ -267,6 +276,7 @@ mod tests {
arch: X86_64, arch: X86_64,
}, },
], ],
variant: None,
repr: "cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64", repr: "cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64",
}, },
}, },
@ -295,6 +305,7 @@ mod tests {
platform_tag: [ platform_tag: [
Any, Any,
], ],
variant: None,
repr: "py3-foo-any", repr: "py3-foo-any",
}, },
}, },
@ -329,6 +340,7 @@ mod tests {
platform_tag: [ platform_tag: [
Any, Any,
], ],
variant: None,
repr: "py2.py3-none-any", repr: "py2.py3-none-any",
}, },
}, },
@ -367,16 +379,6 @@ mod tests {
"#); "#);
} }
#[test]
fn test_error_extra_segment() {
let err = ExpandedTags::parse(vec!["py3-none-any-extra"]).unwrap_err();
insta::assert_debug_snapshot!(err, @r#"
ExtraSegment(
"py3-none-any-extra",
)
"#);
}
#[test] #[test]
fn test_parse_expanded_tag_single_segment() { fn test_parse_expanded_tag_single_segment() {
let result = parse_expanded_tag("py3-none-any"); let result = parse_expanded_tag("py3-none-any");
@ -445,6 +447,7 @@ mod tests {
arch: X86, arch: X86,
}, },
], ],
variant: None,
repr: "cp39.cp310-cp39.cp310-linux_x86_64.linux_i686", repr: "cp39.cp310-cp39.cp310-linux_x86_64.linux_i686",
}, },
} }
@ -487,18 +490,6 @@ mod tests {
"#); "#);
} }
#[test]
fn test_parse_expanded_tag_four_segments() {
let result = parse_expanded_tag("py3-none-any-extra");
assert!(result.is_err());
insta::assert_debug_snapshot!(result.unwrap_err(), @r#"
ExtraSegment(
"py3-none-any-extra",
)
"#);
}
#[test] #[test]
fn test_expanded_tags_ordering() { fn test_expanded_tags_ordering() {
let tags1 = ExpandedTags::parse(vec!["py3-none-any"]).unwrap(); let tags1 = ExpandedTags::parse(vec!["py3-none-any"]).unwrap();

View File

@ -9,6 +9,7 @@ pub use egg::{EggInfoFilename, EggInfoFilenameError};
pub use expanded_tags::{ExpandedTagError, ExpandedTags}; pub use expanded_tags::{ExpandedTagError, ExpandedTags};
pub use extension::{DistExtension, ExtensionError, SourceDistExtension}; pub use extension::{DistExtension, ExtensionError, SourceDistExtension};
pub use source_dist::{SourceDistFilename, SourceDistFilenameError}; pub use source_dist::{SourceDistFilename, SourceDistFilenameError};
pub use variant_label::{InvalidVariantLabel, VariantLabel};
pub use wheel::{WheelFilename, WheelFilenameError}; pub use wheel::{WheelFilename, WheelFilenameError};
mod build_tag; mod build_tag;
@ -17,6 +18,7 @@ mod expanded_tags;
mod extension; mod extension;
mod source_dist; mod source_dist;
mod splitter; mod splitter;
mod variant_label;
mod wheel; mod wheel;
mod wheel_tag; mod wheel_tag;
@ -100,10 +102,20 @@ impl Display for DistFilename {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::WheelFilename; use super::*;
use crate::wheel_tag::WheelTag;
use uv_platform_tags::{AbiTag, LanguageTag, PlatformTag};
#[test] #[test]
fn wheel_filename_size() { fn wheel_filename_size() {
// This value is performance critical
assert_eq!(size_of::<WheelFilename>(), 48); assert_eq!(size_of::<WheelFilename>(), 48);
// Components of the above size
assert_eq!(size_of::<PackageName>(), 8);
assert_eq!(size_of::<Version>(), 16);
assert_eq!(size_of::<WheelTag>(), 24);
assert_eq!(size_of::<LanguageTag>(), 3);
assert_eq!(size_of::<AbiTag>(), 5);
assert_eq!(size_of::<PlatformTag>(), 16);
} }
} }

View File

@ -28,6 +28,7 @@ Ok(
platform_tag: [ platform_tag: [
Any, Any,
], ],
variant: None,
repr: "202206090410-py3-none-any", repr: "202206090410-py3-none-any",
}, },
}, },

View File

@ -38,6 +38,7 @@ Ok(
arch: X86_64, arch: X86_64,
}, },
], ],
variant: None,
repr: "cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64", repr: "cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64",
}, },
}, },

View File

@ -0,0 +1,41 @@
---
source: crates/uv-distribution-filename/src/wheel.rs
expression: "WheelFilename::from_str(\"dummy_project-0.0.1-1234-py3-none-any-36266d4d.whl\")"
snapshot_kind: text
---
Ok(
WheelFilename {
name: PackageName(
"dummy-project",
),
version: "0.0.1",
tags: Large {
large: WheelTagLarge {
build_tag: Some(
BuildTag(
1234,
None,
),
),
python_tag: [
Python {
major: 3,
minor: None,
},
],
abi_tag: [
None,
],
platform_tag: [
Any,
],
variant: Some(
VariantLabel(
"36266d4d",
),
),
repr: "1234-py3-none-any-36266d4d",
},
},
},
)

View File

@ -0,0 +1,36 @@
---
source: crates/uv-distribution-filename/src/wheel.rs
expression: "WheelFilename::from_str(\"dummy_project-0.0.1-py3-none-any-36266d4d.whl\")"
snapshot_kind: text
---
Ok(
WheelFilename {
name: PackageName(
"dummy-project",
),
version: "0.0.1",
tags: Large {
large: WheelTagLarge {
build_tag: None,
python_tag: [
Python {
major: 3,
minor: None,
},
],
abi_tag: [
None,
],
platform_tag: [
Any,
],
variant: Some(
VariantLabel(
"36266d4d",
),
),
repr: "py3-none-any-36266d4d",
},
},
},
)

View File

@ -0,0 +1,82 @@
use serde::{Deserialize, Deserializer};
use std::fmt::Display;
use std::str::FromStr;
use uv_small_str::SmallString;
#[derive(Debug, thiserror::Error)]
pub enum InvalidVariantLabel {
#[error("Invalid character `{invalid}` in variant label, only [a-z0-9._] are allowed: {input}")]
InvalidCharacter { invalid: char, input: String },
#[error("Variant label must be between 1 and 16 characters long, not {length}: {input}")]
InvalidLength { length: usize, input: String },
}
#[derive(
Debug,
Clone,
Eq,
PartialEq,
Hash,
Ord,
PartialOrd,
serde::Serialize,
rkyv::Archive,
rkyv::Deserialize,
rkyv::Serialize,
)]
#[rkyv(derive(Debug))]
#[serde(transparent)]
pub struct VariantLabel(SmallString);
impl VariantLabel {
pub fn as_str(&self) -> &str {
&self.0
}
}
impl FromStr for VariantLabel {
type Err = InvalidVariantLabel;
fn from_str(label: &str) -> Result<Self, Self::Err> {
if let Some(invalid) = label
.chars()
.find(|c| !(c.is_ascii_lowercase() || c.is_ascii_digit() || *c == '.'))
{
if !invalid.is_ascii_lowercase()
&& !invalid.is_ascii_digit()
&& !matches!(invalid, '.' | '_')
{
return Err(InvalidVariantLabel::InvalidCharacter {
invalid,
input: label.to_string(),
});
}
}
// We checked that the label is ASCII only above, so we can use `len()`.
if label.is_empty() || label.len() > 16 {
return Err(InvalidVariantLabel::InvalidLength {
length: label.len(),
input: label.to_string(),
});
}
Ok(Self(SmallString::from(label)))
}
}
impl<'de> Deserialize<'de> for VariantLabel {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
Self::from_str(&s).map_err(serde::de::Error::custom)
}
}
impl Display for VariantLabel {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
Display::fmt(&self.0, f)
}
}

View File

@ -16,7 +16,7 @@ use uv_platform_tags::{
use crate::splitter::MemchrSplitter; use crate::splitter::MemchrSplitter;
use crate::wheel_tag::{WheelTag, WheelTagLarge, WheelTagSmall}; use crate::wheel_tag::{WheelTag, WheelTagLarge, WheelTagSmall};
use crate::{BuildTag, BuildTagError}; use crate::{BuildTag, BuildTagError, InvalidVariantLabel, VariantLabel};
#[derive( #[derive(
Debug, Debug,
@ -113,11 +113,12 @@ impl WheelFilename {
const CACHE_KEY_MAX_LEN: usize = 64; const CACHE_KEY_MAX_LEN: usize = 64;
let full = format!("{}-{}", self.version, self.tags); let full = format!("{}-{}", self.version, self.tags);
if full.len() <= CACHE_KEY_MAX_LEN { if full.len() <= CACHE_KEY_MAX_LEN {
return full; return full;
} }
// Create a digest of the tag string (instead of its individual fields) to retain // Create a digest of the tag string (and variant if it exists) to retain
// compatibility across platforms, Rust versions, etc. // compatibility across platforms, Rust versions, etc.
let digest = cache_digest(&format!("{}", self.tags)); let digest = cache_digest(&format!("{}", self.tags));
@ -132,6 +133,14 @@ impl WheelFilename {
format!("{version}-{digest}") format!("{version}-{digest}")
} }
/// Return the wheel's variant tag, if present.
pub fn variant(&self) -> Option<&VariantLabel> {
match &self.tags {
WheelTag::Small { .. } => None,
WheelTag::Large { large } => large.variant.as_ref(),
}
}
/// Return the wheel's Python tags. /// Return the wheel's Python tags.
pub fn python_tags(&self) -> &[LanguageTag] { pub fn python_tags(&self) -> &[LanguageTag] {
self.tags.python_tags() self.tags.python_tags()
@ -201,24 +210,62 @@ impl WheelFilename {
)); ));
}; };
let (name, version, build_tag, python_tag, abi_tag, platform_tag, is_small) = let (name, version, build_tag, python_tag, abi_tag, platform_tag, variant, is_small) =
if let Some(platform_tag) = splitter.next() { if let Some(platform_tag) = splitter.next() {
if splitter.next().is_some() { // Extract variant from filenames with the format, e.g., `ml_project-0.0.1-py3-none-any-cu128.whl`.
return Err(WheelFilenameError::InvalidWheelFileName( // TODO(charlie): Integrate this into the filename parsing; it's just easier to do it upfront
filename.to_string(), // for now.
"Must have 5 or 6 components, but has more".to_string(), // 7 components: We have both a build tag and a variant tag.
)); if let Some(variant_tag) = splitter.next() {
if splitter.next().is_some() {
return Err(WheelFilenameError::InvalidWheelFileName(
filename.to_string(),
"Must have 5 to 7 components, but has more".to_string(),
));
}
(
&stem[..version],
&stem[version + 1..build_tag_or_python_tag],
Some(&stem[build_tag_or_python_tag + 1..python_tag_or_abi_tag]),
&stem[python_tag_or_abi_tag + 1..abi_tag_or_platform_tag],
&stem[abi_tag_or_platform_tag + 1..platform_tag],
&stem[platform_tag + 1..variant_tag],
Some(&stem[variant_tag + 1..]),
// Always take the slow path if build or variant tag are present.
false,
)
} else {
// 6 components: Determine whether we have a build tag or a variant tag.
if LanguageTag::from_str(
&stem[build_tag_or_python_tag + 1..python_tag_or_abi_tag],
)
.is_ok()
{
(
&stem[..version],
&stem[version + 1..build_tag_or_python_tag],
None,
&stem[build_tag_or_python_tag + 1..python_tag_or_abi_tag],
&stem[python_tag_or_abi_tag + 1..abi_tag_or_platform_tag],
&stem[abi_tag_or_platform_tag + 1..platform_tag],
Some(&stem[platform_tag + 1..]),
// Always take the slow path if a variant tag is present.
false,
)
} else {
(
&stem[..version],
&stem[version + 1..build_tag_or_python_tag],
Some(&stem[build_tag_or_python_tag + 1..python_tag_or_abi_tag]),
&stem[python_tag_or_abi_tag + 1..abi_tag_or_platform_tag],
&stem[abi_tag_or_platform_tag + 1..platform_tag],
&stem[platform_tag + 1..],
None,
// Always take the slow path if a build tag is present.
false,
)
}
} }
(
&stem[..version],
&stem[version + 1..build_tag_or_python_tag],
Some(&stem[build_tag_or_python_tag + 1..python_tag_or_abi_tag]),
&stem[python_tag_or_abi_tag + 1..abi_tag_or_platform_tag],
&stem[abi_tag_or_platform_tag + 1..platform_tag],
&stem[platform_tag + 1..],
// Always take the slow path if a build tag is present.
false,
)
} else { } else {
( (
&stem[..version], &stem[..version],
@ -227,6 +274,7 @@ impl WheelFilename {
&stem[build_tag_or_python_tag + 1..python_tag_or_abi_tag], &stem[build_tag_or_python_tag + 1..python_tag_or_abi_tag],
&stem[python_tag_or_abi_tag + 1..abi_tag_or_platform_tag], &stem[python_tag_or_abi_tag + 1..abi_tag_or_platform_tag],
&stem[abi_tag_or_platform_tag + 1..], &stem[abi_tag_or_platform_tag + 1..],
None,
// Determine whether any of the tag types contain a period, which would indicate // Determine whether any of the tag types contain a period, which would indicate
// that at least one of the tag types includes multiple tags (which in turn // that at least one of the tag types includes multiple tags (which in turn
// necessitates taking the slow path). // necessitates taking the slow path).
@ -244,6 +292,13 @@ impl WheelFilename {
.map_err(|err| WheelFilenameError::InvalidBuildTag(filename.to_string(), err)) .map_err(|err| WheelFilenameError::InvalidBuildTag(filename.to_string(), err))
}) })
.transpose()?; .transpose()?;
let variant = variant
.map(|variant| {
VariantLabel::from_str(variant).map_err(|err| {
WheelFilenameError::InvalidVariantLabel(filename.to_string(), err)
})
})
.transpose()?;
let tags = if let Some(small) = is_small let tags = if let Some(small) = is_small
.then(|| { .then(|| {
@ -274,6 +329,7 @@ impl WheelFilename {
.map(PlatformTag::from_str) .map(PlatformTag::from_str)
.filter_map(Result::ok) .filter_map(Result::ok)
.collect(), .collect(),
variant,
repr: repr.into(), repr: repr.into(),
}), }),
} }
@ -335,6 +391,8 @@ pub enum WheelFilenameError {
InvalidAbiTag(String, ParseAbiTagError), InvalidAbiTag(String, ParseAbiTagError),
#[error("The wheel filename \"{0}\" has an invalid platform tag: {1}")] #[error("The wheel filename \"{0}\" has an invalid platform tag: {1}")]
InvalidPlatformTag(String, ParsePlatformTagError), InvalidPlatformTag(String, ParsePlatformTagError),
#[error("The wheel filename \"{0}\" has an invalid variant label: {1}")]
InvalidVariantLabel(String, InvalidVariantLabel),
#[error("The wheel filename \"{0}\" is missing a language tag")] #[error("The wheel filename \"{0}\" is missing a language tag")]
MissingLanguageTag(String), MissingLanguageTag(String),
#[error("The wheel filename \"{0}\" is missing an ABI tag")] #[error("The wheel filename \"{0}\" is missing an ABI tag")]
@ -388,8 +446,9 @@ mod tests {
#[test] #[test]
fn err_too_many_parts() { fn err_too_many_parts() {
let err = let err =
WheelFilename::from_str("foo-1.2.3-202206090410-py3-none-any-whoops.whl").unwrap_err(); WheelFilename::from_str("foo-1.2.3-202206090410-py3-none-any-whoopsie-whoops.whl")
insta::assert_snapshot!(err, @r###"The wheel filename "foo-1.2.3-202206090410-py3-none-any-whoops.whl" is invalid: Must have 5 or 6 components, but has more"###); .unwrap_err();
insta::assert_snapshot!(err, @r#"The wheel filename "foo-1.2.3-202206090410-py3-none-any-whoopsie-whoops.whl" is invalid: Must have 5 to 7 components, but has more"#);
} }
#[test] #[test]
@ -429,12 +488,23 @@ mod tests {
)); ));
} }
#[test]
fn ok_variant_tag() {
insta::assert_debug_snapshot!(WheelFilename::from_str(
"dummy_project-0.0.1-py3-none-any-36266d4d.whl"
));
insta::assert_debug_snapshot!(WheelFilename::from_str(
"dummy_project-0.0.1-1234-py3-none-any-36266d4d.whl"
));
}
#[test] #[test]
fn from_and_to_string() { fn from_and_to_string() {
let wheel_names = &[ let wheel_names = &[
"django_allauth-0.51.0-py3-none-any.whl", "django_allauth-0.51.0-py3-none-any.whl",
"osm2geojson-0.2.4-py3-none-any.whl", "osm2geojson-0.2.4-py3-none-any.whl",
"numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"dummy_project-0.0.1-py3-none-any-36266d4d.whl",
]; ];
for wheel_name in wheel_names { for wheel_name in wheel_names {
assert_eq!( assert_eq!(
@ -469,5 +539,10 @@ mod tests {
"example-1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.1.2.3.4.5.6.7.8.9.0.1.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" "example-1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.1.2.3.4.5.6.7.8.9.0.1.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
).unwrap(); ).unwrap();
insta::assert_snapshot!(filename.cache_key(), @"1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.1.2-80bf8598e9647cf7"); insta::assert_snapshot!(filename.cache_key(), @"1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.1.2-80bf8598e9647cf7");
// Variant tags should be included in the cache key.
let filename =
WheelFilename::from_str("dummy_project-0.0.1-py3-none-any-36266d4d.whl").unwrap();
insta::assert_snapshot!(filename.cache_key(), @"0.0.1-py3-none-any-36266d4d");
} }
} }

View File

@ -1,6 +1,6 @@
use std::fmt::{Display, Formatter}; use std::fmt::{Display, Formatter};
use crate::BuildTag; use crate::{BuildTag, VariantLabel};
use uv_platform_tags::{AbiTag, LanguageTag, PlatformTag}; use uv_platform_tags::{AbiTag, LanguageTag, PlatformTag};
use uv_small_str::SmallString; use uv_small_str::SmallString;
@ -136,6 +136,8 @@ pub(crate) struct WheelTagLarge {
pub(crate) abi_tag: TagSet<AbiTag>, pub(crate) abi_tag: TagSet<AbiTag>,
/// The platform tag(s), e.g., `none` in `1.2.3-73-py3-none-any`. /// The platform tag(s), e.g., `none` in `1.2.3-73-py3-none-any`.
pub(crate) platform_tag: TagSet<PlatformTag>, pub(crate) platform_tag: TagSet<PlatformTag>,
/// The optional variant tag.
pub(crate) variant: Option<VariantLabel>,
/// The string representation of the tag. /// The string representation of the tag.
/// ///
/// Preserves any unsupported tags that were filtered out when parsing the wheel filename. /// Preserves any unsupported tags that were filtered out when parsing the wheel filename.

View File

@ -30,6 +30,7 @@ uv-platform-tags = { workspace = true }
uv-pypi-types = { workspace = true } uv-pypi-types = { workspace = true }
uv-redacted = { workspace = true } uv-redacted = { workspace = true }
uv-small-str = { workspace = true } uv-small-str = { workspace = true }
uv-variants = { workspace = true }
uv-warnings = { workspace = true } uv-warnings = { workspace = true }
arcstr = { workspace = true } arcstr = { workspace = true }

View File

@ -2,12 +2,13 @@ use std::fmt::{Display, Formatter};
use std::path::PathBuf; use std::path::PathBuf;
use uv_cache_key::{CanonicalUrl, RepositoryUrl}; use uv_cache_key::{CanonicalUrl, RepositoryUrl};
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_pep440::Version; use uv_pep440::Version;
use uv_pypi_types::HashDigest; use uv_pypi_types::HashDigest;
use uv_redacted::DisplaySafeUrl; use uv_redacted::DisplaySafeUrl;
use crate::IndexUrl;
/// A unique identifier for a package. A package can either be identified by a name (e.g., `black`) /// A unique identifier for a package. A package can either be identified by a name (e.g., `black`)
/// or a URL (e.g., `git+https://github.com/psf/black`). /// or a URL (e.g., `git+https://github.com/psf/black`).
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
@ -69,6 +70,25 @@ impl Display for VersionId {
} }
} }
/// A unique identifier for a package version at a specific index (e.g., `black==23.10.0 @ https://pypi.org/simple`).
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct GlobalVersionId {
version: VersionId,
index: IndexUrl,
}
impl GlobalVersionId {
pub fn new(version: VersionId, index: IndexUrl) -> Self {
Self { version, index }
}
}
impl Display for GlobalVersionId {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{} @ {}", self.version, self.index)
}
}
/// A unique resource identifier for the distribution, like a SHA-256 hash of the distribution's /// A unique resource identifier for the distribution, like a SHA-256 hash of the distribution's
/// contents. /// contents.
/// ///

View File

@ -0,0 +1,62 @@
use std::fmt::Display;
use std::str::FromStr;
use uv_distribution_filename::DistFilename;
use uv_normalize::PackageName;
use uv_pep440::Version;
use crate::VariantsJsonFilename;
/// On an index page, there can be wheels, source distributions and `variants.json` files.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub enum IndexEntryFilename {
DistFilename(DistFilename),
VariantJson(VariantsJsonFilename),
}
impl IndexEntryFilename {
pub fn name(&self) -> &PackageName {
match self {
Self::DistFilename(filename) => filename.name(),
Self::VariantJson(variant_json) => &variant_json.name,
}
}
pub fn version(&self) -> &Version {
match self {
Self::DistFilename(filename) => filename.version(),
Self::VariantJson(variant_json) => &variant_json.version,
}
}
/// Parse a filename as either a distribution filename or a `variants.json` filename.
pub fn try_from_normalized_filename(filename: &str) -> Option<Self> {
if let Some(dist_filename) = DistFilename::try_from_normalized_filename(filename) {
Some(Self::DistFilename(dist_filename))
} else if let Ok(variant_json) = VariantsJsonFilename::from_str(filename) {
Some(Self::VariantJson(variant_json))
} else {
None
}
}
/// Parse a filename as either a distribution filename or a `variants.json` filename.
pub fn try_from_filename(filename: &str, package_name: &PackageName) -> Option<Self> {
if let Some(dist_filename) = DistFilename::try_from_filename(filename, package_name) {
Some(Self::DistFilename(dist_filename))
} else if let Ok(variant_json) = VariantsJsonFilename::from_str(filename) {
Some(Self::VariantJson(variant_json))
} else {
None
}
}
}
impl Display for IndexEntryFilename {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::DistFilename(d) => d.fmt(f),
Self::VariantJson(variant_json) => variant_json.fmt(f),
}
}
}

View File

@ -17,6 +17,7 @@ use uv_normalize::PackageName;
use uv_pep440::Version; use uv_pep440::Version;
use uv_pypi_types::{DirectUrl, MetadataError}; use uv_pypi_types::{DirectUrl, MetadataError};
use uv_redacted::DisplaySafeUrl; use uv_redacted::DisplaySafeUrl;
use uv_variants::variants_json::DistInfoVariantsJson;
use crate::{ use crate::{
BuildInfo, DistributionMetadata, InstalledMetadata, InstalledVersion, Name, VersionOrUrlRef, BuildInfo, DistributionMetadata, InstalledMetadata, InstalledVersion, Name, VersionOrUrlRef,
@ -484,6 +485,20 @@ impl InstalledDist {
} }
} }
/// Read the `variant.json` file of the distribution, if it exists.
pub fn read_variant_json(&self) -> Result<Option<DistInfoVariantsJson>, InstalledDistError> {
let path = self.install_path().join("variant.json");
let file = match fs_err::File::open(&path) {
Ok(file) => file,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(err) => return Err(err.into()),
};
let variants_json = serde_json::from_reader::<BufReader<fs_err::File>, DistInfoVariantsJson>(
BufReader::new(file),
)?;
Ok(Some(variants_json))
}
/// Return the supported wheel tags for the distribution from the `WHEEL` file, if available. /// Return the supported wheel tags for the distribution from the `WHEEL` file, if available.
pub fn read_tags(&self) -> Result<Option<&ExpandedTags>, InstalledDistError> { pub fn read_tags(&self) -> Result<Option<&ExpandedTags>, InstalledDistError> {
if let Some(tags) = self.tags_cache.get() { if let Some(tags) = self.tags_cache.get() {

View File

@ -67,6 +67,7 @@ pub use crate::file::*;
pub use crate::hash::*; pub use crate::hash::*;
pub use crate::id::*; pub use crate::id::*;
pub use crate::index::*; pub use crate::index::*;
pub use crate::index_entry::*;
pub use crate::index_name::*; pub use crate::index_name::*;
pub use crate::index_url::*; pub use crate::index_url::*;
pub use crate::installed::*; pub use crate::installed::*;
@ -82,6 +83,7 @@ pub use crate::resolved::*;
pub use crate::specified_requirement::*; pub use crate::specified_requirement::*;
pub use crate::status_code_strategy::*; pub use crate::status_code_strategy::*;
pub use crate::traits::*; pub use crate::traits::*;
pub use crate::variant_json::*;
mod annotation; mod annotation;
mod any; mod any;
@ -98,6 +100,7 @@ mod file;
mod hash; mod hash;
mod id; mod id;
mod index; mod index;
mod index_entry;
mod index_name; mod index_name;
mod index_url; mod index_url;
mod installed; mod installed;
@ -113,6 +116,7 @@ mod resolved;
mod specified_requirement; mod specified_requirement;
mod status_code_strategy; mod status_code_strategy;
mod traits; mod traits;
mod variant_json;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub enum VersionOrUrlRef<'a, T: Pep508Url = VerbatimUrl> { pub enum VersionOrUrlRef<'a, T: Pep508Url = VerbatimUrl> {
@ -631,6 +635,14 @@ impl BuiltDist {
Self::Path(wheel) => &wheel.filename.version, Self::Path(wheel) => &wheel.filename.version,
} }
} }
pub fn wheel_filename(&self) -> &WheelFilename {
match self {
Self::Registry(wheels) => &wheels.best_wheel().filename,
Self::DirectUrl(wheel) => &wheel.filename,
Self::Path(wheel) => &wheel.filename,
}
}
} }
impl SourceDist { impl SourceDist {
@ -1467,6 +1479,34 @@ impl Identifier for BuildableSource<'_> {
} }
} }
/// A built distribution (wheel) that exists in a registry, like `PyPI`.
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub struct RegistryVariantsJson {
pub filename: VariantsJsonFilename,
pub file: Box<File>,
pub index: IndexUrl,
}
impl RegistryVariantsJson {
/// Return the [`GlobalVersionId`] for this registry variants JSON.
pub fn version_id(&self) -> GlobalVersionId {
GlobalVersionId::new(
VersionId::NameVersion(self.filename.name.clone(), self.filename.version.clone()),
self.index.clone(),
)
}
}
impl Identifier for RegistryVariantsJson {
fn distribution_id(&self) -> DistributionId {
self.file.distribution_id()
}
fn resource_id(&self) -> ResourceId {
self.file.resource_id()
}
}
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use crate::{BuiltDist, Dist, RemoteSource, SourceDist, UrlString}; use crate::{BuiltDist, Dist, RemoteSource, SourceDist, UrlString};

View File

@ -9,10 +9,12 @@ use uv_pep440::{Version, VersionSpecifier, VersionSpecifiers};
use uv_pep508::{MarkerExpression, MarkerOperator, MarkerTree, MarkerValueString}; use uv_pep508::{MarkerExpression, MarkerOperator, MarkerTree, MarkerValueString};
use uv_platform_tags::{AbiTag, IncompatibleTag, LanguageTag, PlatformTag, TagPriority, Tags}; use uv_platform_tags::{AbiTag, IncompatibleTag, LanguageTag, PlatformTag, TagPriority, Tags};
use uv_pypi_types::{HashDigest, Yanked}; use uv_pypi_types::{HashDigest, Yanked};
use uv_variants::VariantPriority;
use uv_variants::resolved_variants::ResolvedVariants;
use crate::{ use crate::{
File, InstalledDist, KnownPlatform, RegistryBuiltDist, RegistryBuiltWheel, RegistrySourceDist, File, IndexUrl, InstalledDist, KnownPlatform, RegistryBuiltDist, RegistryBuiltWheel,
ResolvedDistRef, RegistrySourceDist, RegistryVariantsJson, ResolvedDistRef,
}; };
/// A collection of distributions that have been filtered by relevance. /// A collection of distributions that have been filtered by relevance.
@ -26,9 +28,13 @@ struct PrioritizedDistInner {
source: Option<(RegistrySourceDist, SourceDistCompatibility)>, source: Option<(RegistrySourceDist, SourceDistCompatibility)>,
/// The highest-priority wheel index. When present, it is /// The highest-priority wheel index. When present, it is
/// guaranteed to be a valid index into `wheels`. /// guaranteed to be a valid index into `wheels`.
///
/// This wheel may still be incompatible.
best_wheel_index: Option<usize>, best_wheel_index: Option<usize>,
/// The set of all wheels associated with this distribution. /// The set of all wheels associated with this distribution.
wheels: Vec<(RegistryBuiltWheel, WheelCompatibility)>, wheels: Vec<(RegistryBuiltWheel, WheelCompatibility)>,
/// The `variants.json` file associated with the package version.
variants_json: Option<RegistryVariantsJson>,
/// The hashes for each distribution. /// The hashes for each distribution.
hashes: Vec<HashDigest>, hashes: Vec<HashDigest>,
/// The set of supported platforms for the distribution, described in terms of their markers. /// The set of supported platforms for the distribution, described in terms of their markers.
@ -41,6 +47,7 @@ impl Default for PrioritizedDistInner {
source: None, source: None,
best_wheel_index: None, best_wheel_index: None,
wheels: Vec::new(), wheels: Vec::new(),
variants_json: None,
hashes: Vec::new(), hashes: Vec::new(),
markers: MarkerTree::FALSE, markers: MarkerTree::FALSE,
} }
@ -91,6 +98,16 @@ impl CompatibleDist<'_> {
} }
} }
/// Return the index URL for the distribution, if any.
pub fn index(&self) -> Option<&IndexUrl> {
match self {
CompatibleDist::InstalledDist(_) => None,
CompatibleDist::SourceDist { sdist, .. } => Some(&sdist.index),
CompatibleDist::CompatibleWheel { wheel, .. } => Some(&wheel.index),
CompatibleDist::IncompatibleWheel { sdist, .. } => Some(&sdist.index),
}
}
// For installable distributions, return the prioritized distribution it was derived from. // For installable distributions, return the prioritized distribution it was derived from.
pub fn prioritized(&self) -> Option<&PrioritizedDist> { pub fn prioritized(&self) -> Option<&PrioritizedDist> {
match self { match self {
@ -125,6 +142,7 @@ impl IncompatibleDist {
match self { match self {
Self::Wheel(incompatibility) => match incompatibility { Self::Wheel(incompatibility) => match incompatibility {
IncompatibleWheel::NoBinary => format!("has {self}"), IncompatibleWheel::NoBinary => format!("has {self}"),
IncompatibleWheel::Variant => format!("has {self}"),
IncompatibleWheel::Tag(_) => format!("has {self}"), IncompatibleWheel::Tag(_) => format!("has {self}"),
IncompatibleWheel::Yanked(_) => format!("was {self}"), IncompatibleWheel::Yanked(_) => format!("was {self}"),
IncompatibleWheel::ExcludeNewer(ts) => match ts { IncompatibleWheel::ExcludeNewer(ts) => match ts {
@ -153,6 +171,7 @@ impl IncompatibleDist {
match self { match self {
Self::Wheel(incompatibility) => match incompatibility { Self::Wheel(incompatibility) => match incompatibility {
IncompatibleWheel::NoBinary => format!("have {self}"), IncompatibleWheel::NoBinary => format!("have {self}"),
IncompatibleWheel::Variant => format!("have {self}"),
IncompatibleWheel::Tag(_) => format!("have {self}"), IncompatibleWheel::Tag(_) => format!("have {self}"),
IncompatibleWheel::Yanked(_) => format!("were {self}"), IncompatibleWheel::Yanked(_) => format!("were {self}"),
IncompatibleWheel::ExcludeNewer(ts) => match ts { IncompatibleWheel::ExcludeNewer(ts) => match ts {
@ -201,6 +220,7 @@ impl IncompatibleDist {
Some(format!("(e.g., `{tag}`)", tag = tag.cyan())) Some(format!("(e.g., `{tag}`)", tag = tag.cyan()))
} }
IncompatibleWheel::Tag(IncompatibleTag::Invalid) => None, IncompatibleWheel::Tag(IncompatibleTag::Invalid) => None,
IncompatibleWheel::Variant => None,
IncompatibleWheel::NoBinary => None, IncompatibleWheel::NoBinary => None,
IncompatibleWheel::Yanked(..) => None, IncompatibleWheel::Yanked(..) => None,
IncompatibleWheel::ExcludeNewer(..) => None, IncompatibleWheel::ExcludeNewer(..) => None,
@ -218,6 +238,9 @@ impl Display for IncompatibleDist {
match self { match self {
Self::Wheel(incompatibility) => match incompatibility { Self::Wheel(incompatibility) => match incompatibility {
IncompatibleWheel::NoBinary => f.write_str("no source distribution"), IncompatibleWheel::NoBinary => f.write_str("no source distribution"),
IncompatibleWheel::Variant => {
f.write_str("no wheels with a variant supported on the current platform")
}
IncompatibleWheel::Tag(tag) => match tag { IncompatibleWheel::Tag(tag) => match tag {
IncompatibleTag::Invalid => f.write_str("no wheels with valid tags"), IncompatibleTag::Invalid => f.write_str("no wheels with valid tags"),
IncompatibleTag::Python => { IncompatibleTag::Python => {
@ -292,13 +315,20 @@ pub enum PythonRequirementKind {
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub enum WheelCompatibility { pub enum WheelCompatibility {
Incompatible(IncompatibleWheel), Incompatible(IncompatibleWheel),
Compatible(HashComparison, Option<TagPriority>, Option<BuildTag>), Compatible {
hash: HashComparison,
variant_priority: VariantPriority,
tag_priority: Option<TagPriority>,
build_tag: Option<BuildTag>,
},
} }
#[derive(Debug, PartialEq, Eq, Clone)] #[derive(Debug, PartialEq, Eq, Clone)]
pub enum IncompatibleWheel { pub enum IncompatibleWheel {
/// The wheel was published after the exclude newer time. /// The wheel was published after the exclude newer time.
ExcludeNewer(Option<i64>), ExcludeNewer(Option<i64>),
/// The wheel variant does not match the target platform.
Variant,
/// The wheel tags do not match those of the target Python platform. /// The wheel tags do not match those of the target Python platform.
Tag(IncompatibleTag), Tag(IncompatibleTag),
/// The required Python version is not a superset of the target Python version range. /// The required Python version is not a superset of the target Python version range.
@ -346,6 +376,7 @@ impl PrioritizedDist {
markers: implied_markers(&dist.filename), markers: implied_markers(&dist.filename),
best_wheel_index: Some(0), best_wheel_index: Some(0),
wheels: vec![(dist, compatibility)], wheels: vec![(dist, compatibility)],
variants_json: None,
source: None, source: None,
hashes, hashes,
})) }))
@ -362,10 +393,23 @@ impl PrioritizedDist {
best_wheel_index: None, best_wheel_index: None,
wheels: vec![], wheels: vec![],
source: Some((dist, compatibility)), source: Some((dist, compatibility)),
variants_json: None,
hashes, hashes,
})) }))
} }
/// Create a new [`PrioritizedDist`] from the `variants.json`.
pub fn from_variant_json(variant_json: RegistryVariantsJson) -> Self {
Self(Box::new(PrioritizedDistInner {
markers: MarkerTree::TRUE,
best_wheel_index: Some(0),
wheels: vec![],
source: None,
variants_json: Some(variant_json),
hashes: vec![],
}))
}
/// Insert the given built distribution into the [`PrioritizedDist`]. /// Insert the given built distribution into the [`PrioritizedDist`].
pub fn insert_built( pub fn insert_built(
&mut self, &mut self,
@ -419,17 +463,51 @@ impl PrioritizedDist {
} }
} }
pub fn insert_variant_json(&mut self, variant_json: RegistryVariantsJson) {
debug_assert!(
self.0.variants_json.is_none(),
"The variants.json filename is unique"
);
self.0.variants_json = Some(variant_json);
}
/// Return the variants JSON for the distribution, if any.
pub fn variants_json(&self) -> Option<&RegistryVariantsJson> {
self.0.variants_json.as_ref()
}
/// Return the index URL for the distribution, if any.
pub fn index(&self) -> Option<&IndexUrl> {
self.0
.source
.as_ref()
.map(|(sdist, _)| &sdist.index)
.or_else(|| self.0.wheels.first().map(|(wheel, _)| &wheel.index))
}
/// Return the highest-priority distribution for the package version, if any. /// Return the highest-priority distribution for the package version, if any.
pub fn get(&self) -> Option<CompatibleDist<'_>> { pub fn get(&self, allow_all_variants: bool) -> Option<CompatibleDist<'_>> {
let best_wheel = self.0.best_wheel_index.map(|i| &self.0.wheels[i]); let best_wheel = self.0.best_wheel_index.map(|i| &self.0.wheels[i]);
match (&best_wheel, &self.0.source) { match (&best_wheel, &self.0.source) {
// If both are compatible, break ties based on the hash outcome. For example, prefer a // If both are compatible, break ties based on the hash outcome. For example, prefer a
// source distribution with a matching hash over a wheel with a mismatched hash. When // source distribution with a matching hash over a wheel with a mismatched hash. When
// the outcomes are equivalent (e.g., both have a matching hash), prefer the wheel. // the outcomes are equivalent (e.g., both have a matching hash), prefer the wheel.
( (
Some((wheel, WheelCompatibility::Compatible(wheel_hash, tag_priority, ..))), Some((
wheel,
WheelCompatibility::Compatible {
hash: wheel_hash,
tag_priority,
variant_priority,
..
},
)),
Some((sdist, SourceDistCompatibility::Compatible(sdist_hash))), Some((sdist, SourceDistCompatibility::Compatible(sdist_hash))),
) => { ) if matches!(
variant_priority,
VariantPriority::BestVariant | VariantPriority::NonVariant
) || allow_all_variants =>
{
if sdist_hash > wheel_hash { if sdist_hash > wheel_hash {
Some(CompatibleDist::SourceDist { Some(CompatibleDist::SourceDist {
sdist, sdist,
@ -444,7 +522,22 @@ impl PrioritizedDist {
} }
} }
// Prefer the highest-priority, platform-compatible wheel. // Prefer the highest-priority, platform-compatible wheel.
(Some((wheel, WheelCompatibility::Compatible(_, tag_priority, ..))), _) => { (
Some((
wheel,
WheelCompatibility::Compatible {
hash: _,
tag_priority,
variant_priority,
..
},
)),
_,
) if matches!(
variant_priority,
VariantPriority::BestVariant | VariantPriority::NonVariant
) || allow_all_variants =>
{
Some(CompatibleDist::CompatibleWheel { Some(CompatibleDist::CompatibleWheel {
wheel, wheel,
priority: *tag_priority, priority: *tag_priority,
@ -477,6 +570,56 @@ impl PrioritizedDist {
} }
} }
/// Prioritize a matching variant wheel over a matching non-variant wheel.
///
/// Returns `None` or there is no matching variant.
pub fn prioritize_best_variant_wheel(
&self,
resolved_variants: &ResolvedVariants,
) -> Option<Self> {
let mut highest_priority_variant_wheel: Option<(usize, Vec<usize>)> = None;
for (wheel_index, (wheel, compatibility)) in self.wheels().enumerate() {
if !compatibility.is_compatible() {
continue;
}
let Some(variant) = wheel.filename.variant() else {
// The non-variant wheel is already supported
continue;
};
let Some(scores) = resolved_variants.score_variant(variant) else {
continue;
};
if let Some((_, old_scores)) = &highest_priority_variant_wheel {
if &scores > old_scores {
highest_priority_variant_wheel = Some((wheel_index, scores));
}
} else {
highest_priority_variant_wheel = Some((wheel_index, scores));
}
}
if let Some((wheel_index, _)) = highest_priority_variant_wheel {
use owo_colors::OwoColorize;
let inner = PrioritizedDistInner {
best_wheel_index: Some(wheel_index),
..(*self.0).clone()
};
let compatible_wheel = &inner.wheels[wheel_index];
debug!(
"{} {}",
"Using variant wheel".red(),
compatible_wheel.0.filename
);
Some(Self(Box::new(inner)))
} else {
None
}
}
/// Return the incompatibility for the best source distribution, if any. /// Return the incompatibility for the best source distribution, if any.
pub fn incompatible_source(&self) -> Option<&IncompatibleSource> { pub fn incompatible_source(&self) -> Option<&IncompatibleSource> {
self.0 self.0
@ -489,16 +632,24 @@ impl PrioritizedDist {
} }
/// Return the incompatibility for the best wheel, if any. /// Return the incompatibility for the best wheel, if any.
pub fn incompatible_wheel(&self) -> Option<&IncompatibleWheel> { pub fn incompatible_wheel(&self, allow_variants: bool) -> Option<&IncompatibleWheel> {
self.0 self.0
.best_wheel_index .best_wheel_index
.map(|i| &self.0.wheels[i]) .map(|i| &self.0.wheels[i])
.and_then(|(_, compatibility)| match compatibility { .and_then(|(_, compatibility)| match compatibility {
WheelCompatibility::Compatible(_, _, _) => None, WheelCompatibility::Compatible {
variant_priority: VariantPriority::Unknown,
..
} if !allow_variants => Some(&IncompatibleWheel::Variant),
WheelCompatibility::Compatible { .. } => None,
WheelCompatibility::Incompatible(incompatibility) => Some(incompatibility), WheelCompatibility::Incompatible(incompatibility) => Some(incompatibility),
}) })
} }
pub fn wheels(&self) -> impl Iterator<Item = &(RegistryBuiltWheel, WheelCompatibility)> {
self.0.wheels.iter()
}
/// Return the hashes for each distribution. /// Return the hashes for each distribution.
pub fn hashes(&self) -> &[HashDigest] { pub fn hashes(&self) -> &[HashDigest] {
&self.0.hashes &self.0.hashes
@ -665,7 +816,7 @@ impl<'a> CompatibleDist<'a> {
impl WheelCompatibility { impl WheelCompatibility {
/// Return `true` if the distribution is compatible. /// Return `true` if the distribution is compatible.
pub fn is_compatible(&self) -> bool { pub fn is_compatible(&self) -> bool {
matches!(self, Self::Compatible(_, _, _)) matches!(self, Self::Compatible { .. })
} }
/// Return `true` if the distribution is excluded. /// Return `true` if the distribution is excluded.
@ -679,14 +830,30 @@ impl WheelCompatibility {
/// Compatible wheel ordering is determined by tag priority. /// Compatible wheel ordering is determined by tag priority.
pub fn is_more_compatible(&self, other: &Self) -> bool { pub fn is_more_compatible(&self, other: &Self) -> bool {
match (self, other) { match (self, other) {
(Self::Compatible(_, _, _), Self::Incompatible(_)) => true, (Self::Compatible { .. }, Self::Incompatible(..)) => true,
( (
Self::Compatible(hash, tag_priority, build_tag), Self::Compatible {
Self::Compatible(other_hash, other_tag_priority, other_build_tag), hash,
variant_priority,
tag_priority,
build_tag,
},
Self::Compatible {
hash: other_hash,
variant_priority: other_variant_priority,
tag_priority: other_tag_priority,
build_tag: other_build_tag,
},
) => { ) => {
(hash, tag_priority, build_tag) > (other_hash, other_tag_priority, other_build_tag) (hash, variant_priority, tag_priority, build_tag)
> (
other_hash,
other_variant_priority,
other_tag_priority,
other_build_tag,
)
} }
(Self::Incompatible(_), Self::Compatible(_, _, _)) => false, (Self::Incompatible(..), Self::Compatible { .. }) => false,
(Self::Incompatible(incompatibility), Self::Incompatible(other_incompatibility)) => { (Self::Incompatible(incompatibility), Self::Incompatible(other_incompatibility)) => {
incompatibility.is_more_compatible(other_incompatibility) incompatibility.is_more_compatible(other_incompatibility)
} }
@ -768,34 +935,45 @@ impl IncompatibleWheel {
Self::MissingPlatform(_) Self::MissingPlatform(_)
| Self::NoBinary | Self::NoBinary
| Self::RequiresPython(_, _) | Self::RequiresPython(_, _)
| Self::Variant
| Self::Tag(_) | Self::Tag(_)
| Self::Yanked(_) => true, | Self::Yanked(_) => true,
}, },
Self::Variant => match other {
Self::ExcludeNewer(_)
| Self::Tag(_)
| Self::RequiresPython(_, _)
| Self::Yanked(_) => false,
Self::Variant => false,
Self::MissingPlatform(_) | Self::NoBinary => true,
},
Self::Tag(tag_self) => match other { Self::Tag(tag_self) => match other {
Self::ExcludeNewer(_) => false, Self::ExcludeNewer(_) => false,
Self::Tag(tag_other) => tag_self > tag_other, Self::Tag(tag_other) => tag_self > tag_other,
Self::MissingPlatform(_) Self::MissingPlatform(_)
| Self::NoBinary | Self::NoBinary
| Self::RequiresPython(_, _) | Self::RequiresPython(_, _)
| Self::Variant
| Self::Yanked(_) => true, | Self::Yanked(_) => true,
}, },
Self::RequiresPython(_, _) => match other { Self::RequiresPython(_, _) => match other {
Self::ExcludeNewer(_) | Self::Tag(_) => false, Self::ExcludeNewer(_) | Self::Tag(_) => false,
// Version specifiers cannot be reasonably compared // Version specifiers cannot be reasonably compared
Self::RequiresPython(_, _) => false, Self::RequiresPython(_, _) => false,
Self::MissingPlatform(_) | Self::NoBinary | Self::Yanked(_) => true, Self::MissingPlatform(_) | Self::NoBinary | Self::Yanked(_) | Self::Variant => true,
}, },
Self::Yanked(_) => match other { Self::Yanked(_) => match other {
Self::ExcludeNewer(_) | Self::Tag(_) | Self::RequiresPython(_, _) => false, Self::ExcludeNewer(_) | Self::Tag(_) | Self::RequiresPython(_, _) => false,
// Yanks with a reason are more helpful for errors // Yanks with a reason are more helpful for errors
Self::Yanked(yanked_other) => matches!(yanked_other, Yanked::Reason(_)), Self::Yanked(yanked_other) => matches!(yanked_other, Yanked::Reason(_)),
Self::MissingPlatform(_) | Self::NoBinary => true, Self::MissingPlatform(_) | Self::NoBinary | Self::Variant => true,
}, },
Self::NoBinary => match other { Self::NoBinary => match other {
Self::ExcludeNewer(_) Self::ExcludeNewer(_)
| Self::Tag(_) | Self::Tag(_)
| Self::RequiresPython(_, _) | Self::RequiresPython(_, _)
| Self::Yanked(_) => false, | Self::Yanked(_)
| Self::Variant => false,
Self::NoBinary => false, Self::NoBinary => false,
Self::MissingPlatform(_) => true, Self::MissingPlatform(_) => true,
}, },

View File

@ -11,7 +11,8 @@ use uv_git_types::{GitLfs, GitOid, GitReference, GitUrl, GitUrlParseError, OidPa
use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_normalize::{ExtraName, GroupName, PackageName};
use uv_pep440::VersionSpecifiers; use uv_pep440::VersionSpecifiers;
use uv_pep508::{ use uv_pep508::{
MarkerEnvironment, MarkerTree, RequirementOrigin, VerbatimUrl, VersionOrUrl, marker, MarkerEnvironment, MarkerTree, MarkerVariantsEnvironment, RequirementOrigin, VerbatimUrl,
VersionOrUrl, marker,
}; };
use uv_redacted::{DisplaySafeUrl, DisplaySafeUrlError}; use uv_redacted::{DisplaySafeUrl, DisplaySafeUrlError};
@ -69,8 +70,14 @@ impl Requirement {
/// When `env` is `None`, this specifically evaluates all marker /// When `env` is `None`, this specifically evaluates all marker
/// expressions based on the environment to `true`. That is, this provides /// expressions based on the environment to `true`. That is, this provides
/// environment independent marker evaluation. /// environment independent marker evaluation.
pub fn evaluate_markers(&self, env: Option<&MarkerEnvironment>, extras: &[ExtraName]) -> bool { pub fn evaluate_markers(
self.marker.evaluate_optional_environment(env, extras) &self,
env: Option<&MarkerEnvironment>,
variants: &impl MarkerVariantsEnvironment,
extras: &[ExtraName],
) -> bool {
self.marker
.evaluate_optional_environment(env, variants, extras)
} }
/// Returns `true` if the requirement is editable. /// Returns `true` if the requirement is editable.

View File

@ -109,7 +109,7 @@ impl Resolution {
} }
} }
#[derive(Debug, Clone, Hash)] #[derive(Debug, Clone)]
pub enum ResolutionDiagnostic { pub enum ResolutionDiagnostic {
MissingExtra { MissingExtra {
/// The distribution that was requested with a non-existent extra. For example, /// The distribution that was requested with a non-existent extra. For example,

View File

@ -2,14 +2,15 @@ use std::fmt::{Display, Formatter};
use std::path::Path; use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use uv_distribution_filename::WheelFilename;
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_pep440::Version; use uv_pep440::Version;
use uv_pypi_types::Yanked; use uv_pypi_types::Yanked;
use crate::{ use crate::{
BuiltDist, Dist, DistributionId, DistributionMetadata, Identifier, IndexUrl, InstalledDist, BuiltDist, Dist, DistributionId, DistributionMetadata, Identifier, IndexUrl, InstalledDist,
Name, PrioritizedDist, RegistryBuiltWheel, RegistrySourceDist, ResourceId, SourceDist, Name, PrioritizedDist, RegistryBuiltWheel, RegistrySourceDist, RegistryVariantsJson,
VersionOrUrlRef, ResourceId, SourceDist, VersionOrUrlRef,
}; };
/// A distribution that can be used for resolution and installation. /// A distribution that can be used for resolution and installation.
@ -23,6 +24,7 @@ pub enum ResolvedDist {
}, },
Installable { Installable {
dist: Arc<Dist>, dist: Arc<Dist>,
variants_json: Option<Arc<RegistryVariantsJson>>,
version: Option<Version>, version: Option<Version>,
}, },
} }
@ -89,7 +91,7 @@ impl ResolvedDist {
/// Returns the version of the distribution, if available. /// Returns the version of the distribution, if available.
pub fn version(&self) -> Option<&Version> { pub fn version(&self) -> Option<&Version> {
match self { match self {
Self::Installable { version, dist } => dist.version().or(version.as_ref()), Self::Installable { version, dist, .. } => dist.version().or(version.as_ref()),
Self::Installed { dist } => Some(dist.version()), Self::Installed { dist } => Some(dist.version()),
} }
} }
@ -101,6 +103,20 @@ impl ResolvedDist {
Self::Installed { .. } => None, Self::Installed { .. } => None,
} }
} }
// TODO(konsti): That's the wrong "abstraction"
pub fn wheel_filename(&self) -> Option<&WheelFilename> {
match self {
Self::Installed { .. } => {
// TODO(konsti): Support installed dists too
None
}
Self::Installable { dist, .. } => match &**dist {
Dist::Built(dist) => Some(dist.wheel_filename()),
Dist::Source(_) => None,
},
}
}
} }
impl ResolvedDistRef<'_> { impl ResolvedDistRef<'_> {
@ -117,6 +133,9 @@ impl ResolvedDistRef<'_> {
); );
ResolvedDist::Installable { ResolvedDist::Installable {
dist: Arc::new(Dist::Source(SourceDist::Registry(source))), dist: Arc::new(Dist::Source(SourceDist::Registry(source))),
variants_json: prioritized
.variants_json()
.map(|variants_json| Arc::new(variants_json.clone())),
version: Some(sdist.version.clone()), version: Some(sdist.version.clone()),
} }
} }
@ -133,6 +152,9 @@ impl ResolvedDistRef<'_> {
let built = prioritized.built_dist().expect("at least one wheel"); let built = prioritized.built_dist().expect("at least one wheel");
ResolvedDist::Installable { ResolvedDist::Installable {
dist: Arc::new(Dist::Built(BuiltDist::Registry(built))), dist: Arc::new(Dist::Built(BuiltDist::Registry(built))),
variants_json: prioritized
.variants_json()
.map(|variants_json| Arc::new(variants_json.clone())),
version: Some(wheel.filename.version.clone()), version: Some(wheel.filename.version.clone()),
} }
} }

View File

@ -3,7 +3,7 @@ use std::fmt::{Display, Formatter};
use uv_git_types::{GitLfs, GitReference}; use uv_git_types::{GitLfs, GitReference};
use uv_normalize::ExtraName; use uv_normalize::ExtraName;
use uv_pep508::{MarkerEnvironment, MarkerTree, UnnamedRequirement}; use uv_pep508::{MarkerEnvironment, MarkerTree, MarkerVariantsEnvironment, UnnamedRequirement};
use uv_pypi_types::{Hashes, ParsedUrl}; use uv_pypi_types::{Hashes, ParsedUrl};
use crate::{Requirement, RequirementSource, VerbatimParsedUrl}; use crate::{Requirement, RequirementSource, VerbatimParsedUrl};
@ -60,10 +60,17 @@ impl UnresolvedRequirement {
/// that reference the environment as true. In other words, it does /// that reference the environment as true. In other words, it does
/// environment independent expression evaluation. (Which in turn devolves /// environment independent expression evaluation. (Which in turn devolves
/// to "only evaluate marker expressions that reference an extra name.") /// to "only evaluate marker expressions that reference an extra name.")
pub fn evaluate_markers(&self, env: Option<&MarkerEnvironment>, extras: &[ExtraName]) -> bool { pub fn evaluate_markers(
&self,
env: Option<&MarkerEnvironment>,
variants: &impl MarkerVariantsEnvironment,
extras: &[ExtraName],
) -> bool {
match self { match self {
Self::Named(requirement) => requirement.evaluate_markers(env, extras), Self::Named(requirement) => requirement.evaluate_markers(env, variants, extras),
Self::Unnamed(requirement) => requirement.evaluate_optional_environment(env, extras), Self::Unnamed(requirement) => {
requirement.evaluate_optional_environment(env, variants, extras)
}
} }
} }

View File

@ -0,0 +1,92 @@
use std::fmt::Display;
use std::str::FromStr;
use uv_normalize::{InvalidNameError, PackageName};
use uv_pep440::{Version, VersionParseError};
#[derive(Debug, thiserror::Error)]
pub enum VariantsJsonError {
#[error("Invalid `variants.json` filename")]
InvalidFilename,
#[error("Invalid `variants.json` package name: {0}")]
InvalidName(#[from] InvalidNameError),
#[error("Invalid `variants.json` version: {0}")]
InvalidVersion(#[from] VersionParseError),
}
/// A `<name>-<version>-variants.json` filename.
#[derive(
Debug,
Clone,
Hash,
PartialEq,
Eq,
PartialOrd,
Ord,
rkyv::Archive,
rkyv::Deserialize,
rkyv::Serialize,
)]
#[rkyv(derive(Debug))]
pub struct VariantsJsonFilename {
pub name: PackageName,
pub version: Version,
}
impl VariantsJsonFilename {
/// Returns a consistent cache key with a maximum length of 64 characters.
pub fn cache_key(&self) -> String {
const CACHE_KEY_MAX_LEN: usize = 64;
let mut cache_key = self.version.to_string();
if cache_key.len() <= CACHE_KEY_MAX_LEN {
return cache_key;
}
// PANIC SAFETY: version strings can only contain ASCII characters.
cache_key.truncate(CACHE_KEY_MAX_LEN);
let cache_key = cache_key.trim_end_matches(['.', '+']);
cache_key.to_string()
}
}
impl FromStr for VariantsJsonFilename {
type Err = VariantsJsonError;
/// Parse a `<name>-<version>-variants.json` filename.
///
/// name and version must be normalized, i.e., they don't contain dashes.
fn from_str(filename: &str) -> Result<Self, Self::Err> {
let stem = filename
.strip_suffix("-variants.json")
.ok_or(VariantsJsonError::InvalidFilename)?;
let (name, version) = stem
.split_once('-')
.ok_or(VariantsJsonError::InvalidFilename)?;
let name = PackageName::from_str(name)?;
let version = Version::from_str(version)?;
Ok(Self { name, version })
}
}
impl Display for VariantsJsonFilename {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}-{}-variants.json", self.name, self.version)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn variants_json_parsing() {
let variant = VariantsJsonFilename::from_str("numpy-1.21.0-variants.json").unwrap();
assert_eq!(variant.name.as_str(), "numpy");
assert_eq!(variant.version.to_string(), "1.21.0");
}
}

View File

@ -29,18 +29,21 @@ uv-git = { workspace = true }
uv-git-types = { workspace = true } uv-git-types = { workspace = true }
uv-metadata = { workspace = true } uv-metadata = { workspace = true }
uv-normalize = { workspace = true } uv-normalize = { workspace = true }
uv-once-map = { workspace = true }
uv-pep440 = { workspace = true } uv-pep440 = { workspace = true }
uv-pep508 = { workspace = true } uv-pep508 = { workspace = true }
uv-platform-tags = { workspace = true } uv-platform-tags = { workspace = true }
uv-pypi-types = { workspace = true } uv-pypi-types = { workspace = true }
uv-redacted = { workspace = true } uv-redacted = { workspace = true }
uv-types = { workspace = true } uv-types = { workspace = true }
uv-variants = { workspace = true }
uv-workspace = { workspace = true } uv-workspace = { workspace = true }
anyhow = { workspace = true } anyhow = { workspace = true }
either = { workspace = true } either = { workspace = true }
fs-err = { workspace = true } fs-err = { workspace = true }
futures = { workspace = true } futures = { workspace = true }
itertools = { workspace = true }
nanoid = { workspace = true } nanoid = { workspace = true }
owo-colors = { workspace = true } owo-colors = { workspace = true }
reqwest = { workspace = true } reqwest = { workspace = true }

View File

@ -1,16 +1,19 @@
use futures::{FutureExt, StreamExt, TryStreamExt};
use rustc_hash::{FxHashMap, FxHashSet};
use std::ffi::OsString;
use std::future::Future; use std::future::Future;
use std::io;
use std::path::Path; use std::path::Path;
use std::path::PathBuf;
use std::pin::Pin; use std::pin::Pin;
use std::str::FromStr;
use std::sync::Arc; use std::sync::Arc;
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use std::{env, io};
use futures::{FutureExt, TryStreamExt};
use tempfile::TempDir; use tempfile::TempDir;
use tokio::io::{AsyncRead, AsyncSeekExt, ReadBuf}; use tokio::io::{AsyncRead, AsyncSeekExt, ReadBuf};
use tokio::sync::Semaphore; use tokio::sync::Semaphore;
use tokio_util::compat::FuturesAsyncReadCompatExt; use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{Instrument, info_span, instrument, warn}; use tracing::{Instrument, debug, info_span, instrument, trace, warn};
use url::Url; use url::Url;
use uv_cache::{ArchiveId, CacheBucket, CacheEntry, WheelCache}; use uv_cache::{ArchiveId, CacheBucket, CacheEntry, WheelCache};
@ -18,17 +21,24 @@ use uv_cache_info::{CacheInfo, Timestamp};
use uv_client::{ use uv_client::{
CacheControl, CachedClientError, Connectivity, DataWithCachePolicy, RegistryClient, CacheControl, CachedClientError, Connectivity, DataWithCachePolicy, RegistryClient,
}; };
use uv_configuration::BuildOutput;
use uv_distribution_filename::WheelFilename; use uv_distribution_filename::WheelFilename;
use uv_distribution_types::{ use uv_distribution_types::{
BuildInfo, BuildableSource, BuiltDist, Dist, File, HashPolicy, Hashed, IndexUrl, InstalledDist, BuildInfo, BuildableSource, BuiltDist, Dist, File, HashPolicy, Hashed, IndexUrl, InstalledDist,
Name, SourceDist, ToUrlError, Name, RegistryVariantsJson, SourceDist, ToUrlError, VariantsJsonFilename,
}; };
use uv_extract::hash::Hasher; use uv_extract::hash::Hasher;
use uv_fs::write_atomic; use uv_fs::write_atomic;
use uv_pep440::VersionSpecifiers;
use uv_pep508::{MarkerEnvironment, MarkerVariantsUniversal, VariantNamespace, VersionOrUrl};
use uv_platform_tags::Tags; use uv_platform_tags::Tags;
use uv_pypi_types::{HashDigest, HashDigests, PyProjectToml}; use uv_pypi_types::{HashDigest, HashDigests, PyProjectToml};
use uv_redacted::DisplaySafeUrl; use uv_redacted::DisplaySafeUrl;
use uv_types::{BuildContext, BuildStack}; use uv_types::{BuildContext, BuildStack, VariantsTrait};
use uv_variants::VariantProviderOutput;
use uv_variants::resolved_variants::ResolvedVariants;
use uv_variants::variant_lock::{VariantLock, VariantLockProvider, VariantLockResolved};
use uv_variants::variants_json::{Provider, VariantsJsonContent};
use crate::archive::Archive; use crate::archive::Archive;
use crate::metadata::{ArchiveMetadata, Metadata}; use crate::metadata::{ArchiveMetadata, Metadata};
@ -558,6 +568,201 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
.await .await
} }
#[instrument(skip_all, fields(variants_json = %registry_variants_json.filename))]
pub async fn fetch_and_query_variants(
&self,
registry_variants_json: &RegistryVariantsJson,
marker_env: &MarkerEnvironment,
) -> Result<ResolvedVariants, Error> {
let variants_json = self.fetch_variants_json(registry_variants_json).await?;
let resolved_variants = self
.query_variant_providers(variants_json, marker_env, &registry_variants_json.filename)
.await?;
Ok(resolved_variants)
}
/// Fetch the variants.json contents from a URL (cached) or from a path.
async fn fetch_variants_json(
&self,
variants_json: &RegistryVariantsJson,
) -> Result<VariantsJsonContent, Error> {
Ok(self
.client
.managed(|client| client.fetch_variants_json(variants_json))
.await?)
}
async fn query_variant_providers(
&self,
variants_json: VariantsJsonContent,
marker_env: &MarkerEnvironment,
debug_filename: &VariantsJsonFilename,
) -> Result<ResolvedVariants, Error> {
// TODO: parse_boolish_environment_variable
let locked_and_inferred =
env::var_os("UV_VARIANT_LOCK_INCOMPLETE").is_some_and(|var| var == "1");
// TODO(konsti): Integrate this properly, and add this to the CLI.
let variant_lock = if let Some(variant_lock_path) = env::var_os("UV_VARIANT_LOCK") {
let variant_lock: VariantLock = toml::from_slice(
&fs_err::read(&variant_lock_path).map_err(Error::VariantLockRead)?,
)
.map_err(|err| Error::VariantLockParse(PathBuf::from(&variant_lock_path), err))?;
// TODO(konsti): If parsing fails, check the version
if !VersionSpecifiers::from_str(">=0.1,<0.2")
.unwrap()
.contains(&variant_lock.metadata.version)
{
return Err(Error::VariantLockVersion(
PathBuf::from(variant_lock_path),
variant_lock.metadata.version,
));
}
Some((variant_lock_path, variant_lock))
} else {
None
};
// Query the install time provider.
// TODO(konsti): Don't use threads if we're fully static.
let mut disabled_namespaces = FxHashSet::default();
let mut resolved_namespaces: FxHashMap<VariantNamespace, Arc<VariantProviderOutput>> =
futures::stream::iter(variants_json.providers.iter().filter(|(_, provider)| {
provider.install_time.unwrap_or(true)
&& !provider.optional
&& provider
.enable_if
.evaluate(marker_env, &MarkerVariantsUniversal, &[])
}))
.map(|(name, provider)| {
self.resolve_provider(locked_and_inferred, variant_lock.as_ref(), name, provider)
})
// TODO(konsti): Buffer size
.buffered(8)
.try_collect()
.await?;
// "Query" the static providers
for (namespace, provider) in &variants_json.providers {
// Track disabled namespaces for consistency checks.
if !provider
.enable_if
.evaluate(marker_env, &MarkerVariantsUniversal, &[])
|| provider.optional
{
disabled_namespaces.insert(namespace.clone());
continue;
}
if provider.install_time.unwrap_or(true) {
continue;
}
let Some(features) = variants_json
.static_properties
.as_ref()
.and_then(|static_properties| static_properties.get(namespace))
else {
warn!(
"Missing namespace {namespace} in default properties for {}=={}",
debug_filename.name, debug_filename.version
);
continue;
};
resolved_namespaces.insert(
namespace.clone(),
Arc::new(VariantProviderOutput {
namespace: namespace.clone(),
features: features.clone().into_iter().collect(),
}),
);
}
Ok(ResolvedVariants {
variants_json,
resolved_namespaces,
disabled_namespaces,
})
}
async fn resolve_provider(
&self,
locked_and_inferred: bool,
variant_lock: Option<&(OsString, VariantLock)>,
name: &VariantNamespace,
provider: &Provider,
) -> Result<(VariantNamespace, Arc<VariantProviderOutput>), Error> {
if let Some((variant_lock_path, variant_lock)) = &variant_lock {
if let Some(static_provider) = variant_lock
.provider
.iter()
.find(|static_provider| satisfies_provider_requires(provider, static_provider))
{
Ok((
static_provider.namespace.clone(),
Arc::new(VariantProviderOutput {
namespace: static_provider.namespace.clone(),
features: static_provider.properties.clone().into_iter().collect(),
}),
))
} else if locked_and_inferred {
let config = self.query_variant_provider(name, provider).await?;
Ok((config.namespace.clone(), config))
} else {
Err(Error::VariantLockMissing {
variant_lock: PathBuf::from(variant_lock_path),
requires: provider.requires.clone().unwrap_or_default(),
plugin_api: provider.plugin_api.clone().unwrap_or_default().clone(),
})
}
} else {
let config = self.query_variant_provider(name, provider).await?;
Ok((config.namespace.clone(), config))
}
}
async fn query_variant_provider(
&self,
name: &VariantNamespace,
provider: &Provider,
) -> Result<Arc<VariantProviderOutput>, Error> {
let config = if self.build_context.variants().register(provider.clone()) {
debug!("Querying provider `{name}` for variants");
// TODO(konsti): That's not spec compliant
let backend_name = provider.plugin_api.clone().unwrap_or(name.to_string());
let builder = self
.build_context
.setup_variants(backend_name, provider, BuildOutput::Debug)
.await?;
let config = builder.query().await?;
trace!(
"Found namespace {} with configs {:?}",
config.namespace, config
);
let config = Arc::new(config);
self.build_context
.variants()
.done(provider.clone(), config.clone());
config
} else {
debug!("Reading provider `{name}` from in-memory cache");
self.build_context
.variants()
.wait(provider)
.await
.expect("missing value for registered task")
};
if &config.namespace != name {
return Err(Error::WheelVariantNamespaceMismatch {
declared: name.clone(),
actual: config.namespace.clone(),
});
}
Ok(config)
}
/// Stream a wheel from a URL, unzipping it into the cache as it's downloaded. /// Stream a wheel from a URL, unzipping it into the cache as it's downloaded.
async fn stream_wheel( async fn stream_wheel(
&self, &self,
@ -1343,6 +1548,46 @@ fn add_tar_zst_extension(mut url: DisplaySafeUrl) -> DisplaySafeUrl {
url url
} }
fn satisfies_provider_requires(
requested_provider: &Provider,
static_provider: &VariantLockProvider,
) -> bool {
// TODO(konsti): Correct plugin_api inference.
if static_provider.plugin_api.clone().or(static_provider
.resolved
.first()
.map(|resolved| resolved.name().to_string()))
!= requested_provider.plugin_api.clone().or(static_provider
.resolved
.first()
.map(|resolved| resolved.name().to_string()))
{
return false;
}
let Some(requires) = &requested_provider.requires else {
return true;
};
requires.iter().all(|requested| {
static_provider.resolved.iter().any(|resolved| {
// We ignore extras for simplicity.
&requested.name == resolved.name()
&& match (&requested.version_or_url, resolved) {
(None, _) => true,
(
Some(VersionOrUrl::VersionSpecifier(requested)),
VariantLockResolved::Version(_name, resolved),
) => requested.contains(resolved),
(
Some(VersionOrUrl::Url(resolved_url)),
VariantLockResolved::Url(_name, url),
) => resolved_url == &**url,
// We don't support URL<->version matching
_ => false,
}
})
})
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;

View File

@ -1,5 +1,6 @@
use std::path::PathBuf; use std::path::PathBuf;
use itertools::Itertools;
use owo_colors::OwoColorize; use owo_colors::OwoColorize;
use tokio::task::JoinError; use tokio::task::JoinError;
use zip::result::ZipError; use zip::result::ZipError;
@ -12,7 +13,8 @@ use uv_fs::Simplified;
use uv_git::GitError; use uv_git::GitError;
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_pep440::{Version, VersionSpecifiers}; use uv_pep440::{Version, VersionSpecifiers};
use uv_pypi_types::{HashAlgorithm, HashDigest}; use uv_pep508::{Requirement, VariantNamespace};
use uv_pypi_types::{HashAlgorithm, HashDigest, VerbatimParsedUrl};
use uv_redacted::DisplaySafeUrl; use uv_redacted::DisplaySafeUrl;
use uv_types::AnyErrorBuild; use uv_types::AnyErrorBuild;
@ -75,6 +77,32 @@ pub enum Error {
filename: Version, filename: Version,
metadata: Version, metadata: Version,
}, },
#[error(
"Package {name} has no matching wheel for the current platform, but has the following variants: {variants}"
)]
WheelVariantMismatch { name: PackageName, variants: String },
#[error("Provider plugin is declared to use namespace {declared} but uses namespace {actual}")]
WheelVariantNamespaceMismatch {
declared: VariantNamespace,
actual: VariantNamespace,
},
#[error("Failed to read variant lock")]
VariantLockRead(#[source] std::io::Error),
#[error("Variant lock has an unsupported format: {}", _0.user_display())]
VariantLockParse(PathBuf, #[source] toml::de::Error),
#[error("Variant lock has an unsupported version {}, only version 0.1 is supported: {}", _1, _0.user_display())]
VariantLockVersion(PathBuf, Version),
#[error(
"Variant lock is missing a matching provider and `UV_VARIANT_LOCK_INCOMPLETE` is not set\n variant lock: {}\n requires: `{}`\n plugin-api: {}",
variant_lock.user_display(),
requires.iter().join("`, `"),
plugin_api
)]
VariantLockMissing {
variant_lock: PathBuf,
requires: Vec<Requirement<VerbatimParsedUrl>>,
plugin_api: String,
},
#[error("Failed to parse metadata from built wheel")] #[error("Failed to parse metadata from built wheel")]
Metadata(#[from] uv_pypi_types::MetadataError), Metadata(#[from] uv_pypi_types::MetadataError),
#[error("Failed to read metadata: `{}`", _0.user_display())] #[error("Failed to read metadata: `{}`", _0.user_display())]

View File

@ -9,6 +9,7 @@ pub use metadata::{
}; };
pub use reporter::Reporter; pub use reporter::Reporter;
pub use source::prune; pub use source::prune;
pub use variants::{PackageVariantCache, resolve_variants};
mod archive; mod archive;
mod distribution_database; mod distribution_database;
@ -18,3 +19,4 @@ mod index;
mod metadata; mod metadata;
mod reporter; mod reporter;
mod source; mod source;
mod variants;

View File

@ -0,0 +1,205 @@
use std::hash::BuildHasherDefault;
use std::sync::Arc;
use futures::{StreamExt, TryStreamExt};
use itertools::Itertools;
use owo_colors::OwoColorize;
use rustc_hash::{FxHashMap, FxHasher};
use tracing::{debug, trace};
use uv_distribution_filename::BuildTag;
use uv_distribution_types::{
BuiltDist, Dist, DistributionId, GlobalVersionId, Identifier, Name, Node, RegistryBuiltDist,
RegistryVariantsJson, Resolution, ResolvedDist,
};
use uv_once_map::OnceMap;
use uv_platform_tags::{TagCompatibility, Tags};
use uv_pypi_types::ResolverMarkerEnvironment;
use uv_types::BuildContext;
use uv_variants::resolved_variants::ResolvedVariants;
use crate::{DistributionDatabase, Error};
type FxOnceMap<K, V> = OnceMap<K, V, BuildHasherDefault<FxHasher>>;
/// An in-memory cache from package to resolved variants.
#[derive(Default)]
pub struct PackageVariantCache(FxOnceMap<GlobalVersionId, Arc<ResolvedVariants>>);
impl std::ops::Deref for PackageVariantCache {
type Target = FxOnceMap<GlobalVersionId, Arc<ResolvedVariants>>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
/// Resolve all variants for the given resolution.
pub async fn resolve_variants<Context: BuildContext>(
resolution: Resolution,
marker_env: &ResolverMarkerEnvironment,
distribution_database: DistributionDatabase<'_, Context>,
cache: &PackageVariantCache,
tags: &Tags,
) -> Result<Resolution, Error> {
// Fetch variants.json and then query providers, running in parallel for all distributions.
let dist_resolved_variants: FxHashMap<GlobalVersionId, Arc<ResolvedVariants>> =
futures::stream::iter(
resolution
.graph()
.node_weights()
.filter_map(|node| extract_variants(node)),
)
.map(async |(variants_json, ..)| {
let id = variants_json.version_id();
let resolved_variants = if cache.register(id.clone()) {
let resolved_variants = distribution_database
.fetch_and_query_variants(variants_json, marker_env)
.await?;
let resolved_variants = Arc::new(resolved_variants);
cache.done(id.clone(), resolved_variants.clone());
resolved_variants
} else {
cache
.wait(&id)
.await
.expect("missing value for registered task")
};
Ok::<_, Error>((id, resolved_variants))
})
// TODO(konsti): Buffer size
.buffered(8)
.try_collect()
.await?;
// Determine modification to the resolutions to select variant wheels, or error if there
// is no matching variant wheel and no matching non-variant wheel.
let mut new_best_wheel_index: FxHashMap<DistributionId, usize> = FxHashMap::default();
for node in resolution.graph().node_weights() {
let Some((json, dist)) = extract_variants(node) else {
continue;
};
let resolved_variants = &dist_resolved_variants[&json.version_id()];
// Select best wheel
let mut highest_priority_variant_wheel: Option<(
usize,
Vec<usize>,
TagCompatibility,
Option<&BuildTag>,
)> = None;
for (wheel_index, wheel) in dist.wheels.iter().enumerate() {
let build_tag = wheel.filename.build_tag();
let compatibility = wheel.filename.compatibility(tags);
if !compatibility.is_compatible() {
continue;
}
let Some(variant) = wheel.filename.variant() else {
// The non-variant wheel is already supported
continue;
};
let Some(scores) = resolved_variants.score_variant(variant) else {
continue;
};
if let Some((_, old_scores, old_compatibility, old_build_tag)) =
&highest_priority_variant_wheel
{
if (&scores, &compatibility, &build_tag)
> (old_scores, old_compatibility, old_build_tag)
{
highest_priority_variant_wheel =
Some((wheel_index, scores, compatibility, build_tag));
}
} else {
highest_priority_variant_wheel =
Some((wheel_index, scores, compatibility, build_tag));
}
}
// Determine if we need to modify the resolution
if let Some((wheel_index, ..)) = highest_priority_variant_wheel {
debug!(
"{} for {}: {}",
"Using variant wheel".red(),
dist.name(),
dist.wheels[wheel_index].filename,
);
new_best_wheel_index.insert(dist.distribution_id(), wheel_index);
} else if dist.best_wheel().filename.variant().is_some() {
return Err(Error::WheelVariantMismatch {
name: dist.name().clone(),
variants: dist
.wheels
.iter()
.filter_map(|wheel| wheel.filename.variant())
.join(", "),
});
} else {
trace!(
"No matching variant wheel, but matching non-variant wheel for {}",
dist.name()
);
}
}
let resolution = resolution.map(|dist| {
let ResolvedDist::Installable {
dist,
version,
variants_json,
} = dist
else {
return None;
};
let Dist::Built(BuiltDist::Registry(dist)) = &**dist else {
return None;
};
// Check whether there is a matching variant wheel we want to use instead of the default.
let best_wheel_index = new_best_wheel_index.get(&dist.distribution_id())?;
Some(ResolvedDist::Installable {
dist: Arc::new(Dist::Built(BuiltDist::Registry(RegistryBuiltDist {
wheels: dist.wheels.clone(),
best_wheel_index: *best_wheel_index,
sdist: dist.sdist.clone(),
}))),
variants_json: variants_json.clone(),
version: version.clone(),
})
});
Ok(resolution)
}
fn extract_variants(node: &Node) -> Option<(&RegistryVariantsJson, &RegistryBuiltDist)> {
let Node::Dist { dist, .. } = node else {
// The root node has no variants
return None;
};
let ResolvedDist::Installable {
dist,
variants_json,
..
} = dist
else {
// TODO(konsti): Installed dists? Or is that not a thing here?
return None;
};
let Some(variants_json) = variants_json else {
return None;
};
let Dist::Built(BuiltDist::Registry(dist)) = &**dist else {
return None;
};
if !dist
.wheels
.iter()
.any(|wheel| wheel.filename.variant().is_some())
{
return None;
}
Some((variants_json, dist))
}

View File

@ -14,7 +14,7 @@ use uv_distribution_filename::WheelFilename;
use uv_distribution_types::{ use uv_distribution_types::{
BuiltDist, CachedDirectUrlDist, CachedDist, ConfigSettings, Dist, Error, ExtraBuildRequires, BuiltDist, CachedDirectUrlDist, CachedDist, ConfigSettings, Dist, Error, ExtraBuildRequires,
ExtraBuildVariables, Hashed, IndexLocations, InstalledDist, Name, PackageConfigSettings, ExtraBuildVariables, Hashed, IndexLocations, InstalledDist, Name, PackageConfigSettings,
RequirementSource, Resolution, ResolvedDist, SourceDist, RemoteSource, RequirementSource, Resolution, ResolvedDist, SourceDist,
}; };
use uv_fs::Simplified; use uv_fs::Simplified;
use uv_normalize::PackageName; use uv_normalize::PackageName;
@ -208,7 +208,10 @@ impl<'a> Planner<'a> {
} }
Some(&entry.dist) Some(&entry.dist)
}) { }) {
debug!("Registry requirement already cached: {distribution}"); debug!(
"Registry requirement already cached: {distribution} ({})",
wheel.best_wheel().filename
);
cached.push(CachedDist::Registry(distribution.clone())); cached.push(CachedDist::Registry(distribution.clone()));
continue; continue;
} }
@ -494,7 +497,11 @@ impl<'a> Planner<'a> {
} }
} }
debug!("Identified uncached distribution: {dist}"); if let Ok(filename) = dist.filename() {
debug!("Identified uncached distribution: {dist} ({filename})");
} else {
debug!("Identified uncached distribution: {dist}");
}
remote.push(dist.clone()); remote.push(dist.clone());
} }

View File

@ -15,7 +15,7 @@ use uv_distribution_types::{
use uv_fs::Simplified; use uv_fs::Simplified;
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_pep440::{Version, VersionSpecifiers}; use uv_pep440::{Version, VersionSpecifiers};
use uv_pep508::VersionOrUrl; use uv_pep508::{MarkerVariantsUniversal, VersionOrUrl};
use uv_platform_tags::Tags; use uv_platform_tags::Tags;
use uv_pypi_types::{ResolverMarkerEnvironment, VerbatimParsedUrl}; use uv_pypi_types::{ResolverMarkerEnvironment, VerbatimParsedUrl};
use uv_python::{Interpreter, PythonEnvironment}; use uv_python::{Interpreter, PythonEnvironment};
@ -265,7 +265,7 @@ impl SitePackages {
// Verify that the dependencies are installed. // Verify that the dependencies are installed.
for dependency in &metadata.requires_dist { for dependency in &metadata.requires_dist {
if !dependency.evaluate_markers(markers, &[]) { if !dependency.evaluate_markers(markers, &MarkerVariantsUniversal, &[]) {
continue; continue;
} }
@ -454,14 +454,15 @@ impl SitePackages {
for requirement in requirements { for requirement in requirements {
if let Some(r#overrides) = overrides.get(&requirement.name) { if let Some(r#overrides) = overrides.get(&requirement.name) {
for dependency in r#overrides { for dependency in r#overrides {
if dependency.evaluate_markers(Some(markers), &[]) { if dependency.evaluate_markers(Some(markers), &MarkerVariantsUniversal, &[]) {
if seen.insert((*dependency).clone()) { if seen.insert((*dependency).clone()) {
stack.push(Cow::Borrowed(*dependency)); stack.push(Cow::Borrowed(*dependency));
} }
} }
} }
} else { } else {
if requirement.evaluate_markers(Some(markers), &[]) { // TODO(konsti): Evaluate variants
if requirement.evaluate_markers(Some(markers), &MarkerVariantsUniversal, &[]) {
if seen.insert(requirement.clone()) { if seen.insert(requirement.clone()) {
stack.push(Cow::Borrowed(requirement)); stack.push(Cow::Borrowed(requirement));
} }
@ -480,7 +481,7 @@ impl SitePackages {
} }
[distribution] => { [distribution] => {
// Validate that the requirement is satisfied. // Validate that the requirement is satisfied.
if requirement.evaluate_markers(Some(markers), &[]) { if requirement.evaluate_markers(Some(markers), &MarkerVariantsUniversal, &[]) {
match RequirementSatisfaction::check( match RequirementSatisfaction::check(
name, name,
distribution, distribution,
@ -503,7 +504,8 @@ impl SitePackages {
// Validate that the installed version satisfies the constraints. // Validate that the installed version satisfies the constraints.
for constraint in constraints.get(name).into_iter().flatten() { for constraint in constraints.get(name).into_iter().flatten() {
if constraint.evaluate_markers(Some(markers), &[]) { if constraint.evaluate_markers(Some(markers), &MarkerVariantsUniversal, &[])
{
match RequirementSatisfaction::check( match RequirementSatisfaction::check(
name, name,
distribution, distribution,
@ -537,14 +539,22 @@ impl SitePackages {
let dependency = Requirement::from(dependency.clone()); let dependency = Requirement::from(dependency.clone());
if let Some(r#overrides) = overrides.get(&dependency.name) { if let Some(r#overrides) = overrides.get(&dependency.name) {
for dependency in r#overrides { for dependency in r#overrides {
if dependency.evaluate_markers(Some(markers), &requirement.extras) { if dependency.evaluate_markers(
Some(markers),
&MarkerVariantsUniversal,
&requirement.extras,
) {
if seen.insert((*dependency).clone()) { if seen.insert((*dependency).clone()) {
stack.push(Cow::Borrowed(*dependency)); stack.push(Cow::Borrowed(*dependency));
} }
} }
} }
} else { } else {
if dependency.evaluate_markers(Some(markers), &requirement.extras) { if dependency.evaluate_markers(
Some(markers),
&MarkerVariantsUniversal,
&requirement.extras,
) {
if seen.insert(dependency.clone()) { if seen.insert(dependency.clone()) {
stack.push(Cow::Owned(dependency)); stack.push(Cow::Owned(dependency));
} }

View File

@ -36,7 +36,9 @@ pub use crate::marker::{
ContainsMarkerTree, ExtraMarkerTree, ExtraOperator, InMarkerTree, MarkerEnvironment, ContainsMarkerTree, ExtraMarkerTree, ExtraOperator, InMarkerTree, MarkerEnvironment,
MarkerEnvironmentBuilder, MarkerExpression, MarkerOperator, MarkerTree, MarkerTreeContents, MarkerEnvironmentBuilder, MarkerExpression, MarkerOperator, MarkerTree, MarkerTreeContents,
MarkerTreeKind, MarkerValue, MarkerValueExtra, MarkerValueList, MarkerValueString, MarkerTreeKind, MarkerValue, MarkerValueExtra, MarkerValueList, MarkerValueString,
MarkerValueVersion, MarkerWarningKind, StringMarkerTree, StringVersion, VersionMarkerTree, MarkerValueVersion, MarkerVariantsEnvironment, MarkerVariantsUniversal, MarkerWarningKind,
StringMarkerTree, StringVersion, VariantFeature, VariantNamespace, VariantValue,
VersionMarkerTree,
}; };
pub use crate::origin::RequirementOrigin; pub use crate::origin::RequirementOrigin;
#[cfg(feature = "non-pep508-extensions")] #[cfg(feature = "non-pep508-extensions")]
@ -50,6 +52,8 @@ pub use crate::verbatim_url::{
pub use uv_pep440; pub use uv_pep440;
use uv_pep440::{VersionSpecifier, VersionSpecifiers}; use uv_pep440::{VersionSpecifier, VersionSpecifiers};
use crate::marker::VariantParseError;
mod cursor; mod cursor;
pub mod marker; pub mod marker;
mod origin; mod origin;
@ -82,6 +86,20 @@ pub enum Pep508ErrorSource<T: Pep508Url = VerbatimUrl> {
/// The version requirement is not supported. /// The version requirement is not supported.
#[error("{0}")] #[error("{0}")]
UnsupportedRequirement(String), UnsupportedRequirement(String),
/// The operator is not supported with the variant marker.
#[error(
"The operator {0} is not supported with the marker {1}, only the `in` and `not in` operators are supported"
)]
ListOperator(MarkerOperator, MarkerValueList),
/// The value is not a quoted string.
#[error("Only quoted strings are supported with the variant marker {1}, not {0}")]
ListValue(MarkerValue, MarkerValueList),
/// The variant marker is on the left hand side of the expression.
#[error("The marker {0} must be on the right hand side of the expression")]
ListLValue(MarkerValueList),
/// A variant segment uses invalid characters.
#[error(transparent)]
InvalidVariantSegment(VariantParseError),
} }
impl<T: Pep508Url> Display for Pep508Error<T> { impl<T: Pep508Url> Display for Pep508Error<T> {
@ -298,8 +316,13 @@ impl<T: Pep508Url> CacheKey for Requirement<T> {
impl<T: Pep508Url> Requirement<T> { impl<T: Pep508Url> Requirement<T> {
/// Returns whether the markers apply for the given environment /// Returns whether the markers apply for the given environment
pub fn evaluate_markers(&self, env: &MarkerEnvironment, extras: &[ExtraName]) -> bool { pub fn evaluate_markers(
self.marker.evaluate(env, extras) &self,
env: &MarkerEnvironment,
variants: &impl MarkerVariantsEnvironment,
extras: &[ExtraName],
) -> bool {
self.marker.evaluate(env, variants, extras)
} }
/// Return the requirement with an additional marker added, to require the given extra. /// Return the requirement with an additional marker added, to require the given extra.

View File

@ -46,6 +46,7 @@
//! merged to be applied globally. //! merged to be applied globally.
use std::cmp::Ordering; use std::cmp::Ordering;
use std::collections::BTreeSet;
use std::fmt; use std::fmt;
use std::ops::Bound; use std::ops::Bound;
use std::sync::{LazyLock, Mutex, MutexGuard}; use std::sync::{LazyLock, Mutex, MutexGuard};
@ -581,6 +582,43 @@ impl InternerGuard<'_> {
} }
} }
/// Contract: The type of variable remains the same.
pub(crate) fn edit_variable(
&mut self,
i: NodeId,
f: &impl Fn(&Variable) -> Option<Variable>,
) -> NodeId {
if matches!(i, NodeId::TRUE | NodeId::FALSE) {
return i;
}
// Restrict all nodes recursively.
let node = self.shared.node(i);
let children = node.children.map(i, |node| self.edit_variable(node, f));
if let Some(var) = f(&node.var) {
self.create_node(var, children)
} else {
self.create_node(node.var.clone(), children)
}
}
pub(crate) fn collect_variant_bases(&mut self, i: NodeId, bases: &mut BTreeSet<String>) {
if matches!(i, NodeId::TRUE | NodeId::FALSE) {
return;
}
// Restrict all nodes recursively.
let node = self.shared.node(i);
if let Some(base) = node.var.variant_base() {
bases.insert(base.to_string());
}
for child in node.children.nodes() {
self.collect_variant_bases(child, bases);
}
}
/// Returns a new tree where the only nodes remaining are `extra` nodes. /// Returns a new tree where the only nodes remaining are `extra` nodes.
/// ///
/// If there are no extra nodes, then this returns a tree that is always /// If there are no extra nodes, then this returns a tree that is always
@ -1072,6 +1110,17 @@ impl Variable {
}; };
marker.is_conflicting() marker.is_conflicting()
} }
fn variant_base(&self) -> Option<&str> {
match self {
Self::List(
CanonicalMarkerListPair::VariantNamespaces { base, .. }
| CanonicalMarkerListPair::VariantFeatures { base, .. }
| CanonicalMarkerListPair::VariantProperties { base, .. },
) => base.as_deref(),
_ => None,
}
}
} }
/// A decision node in an Algebraic Decision Diagram. /// A decision node in an Algebraic Decision Diagram.

View File

@ -41,8 +41,8 @@ impl MarkerEnvironment {
} }
/// Returns of the stringly typed value of the key in the current environment /// Returns of the stringly typed value of the key in the current environment
pub fn get_string(&self, key: CanonicalMarkerValueString) -> &str { pub fn get_string(&self, key: CanonicalMarkerValueString) -> Option<&str> {
match key { Some(match key {
CanonicalMarkerValueString::ImplementationName => self.implementation_name(), CanonicalMarkerValueString::ImplementationName => self.implementation_name(),
CanonicalMarkerValueString::OsName => self.os_name(), CanonicalMarkerValueString::OsName => self.os_name(),
CanonicalMarkerValueString::PlatformMachine => self.platform_machine(), CanonicalMarkerValueString::PlatformMachine => self.platform_machine(),
@ -53,7 +53,8 @@ impl MarkerEnvironment {
CanonicalMarkerValueString::PlatformSystem => self.platform_system(), CanonicalMarkerValueString::PlatformSystem => self.platform_system(),
CanonicalMarkerValueString::PlatformVersion => self.platform_version(), CanonicalMarkerValueString::PlatformVersion => self.platform_version(),
CanonicalMarkerValueString::SysPlatform => self.sys_platform(), CanonicalMarkerValueString::SysPlatform => self.sys_platform(),
} CanonicalMarkerValueString::VariantLabel => return None,
})
} }
} }

View File

@ -1,8 +1,8 @@
use std::fmt::{Display, Formatter}; use std::fmt::{Display, Formatter};
use uv_normalize::{ExtraName, GroupName}; use uv_normalize::{ExtraName, GroupName};
use crate::marker::tree::MarkerValueList; use crate::marker::tree::MarkerValueList;
use crate::marker::{VariantFeature, VariantNamespace, VariantValue};
use crate::{MarkerValueExtra, MarkerValueString, MarkerValueVersion}; use crate::{MarkerValueExtra, MarkerValueString, MarkerValueVersion};
/// Those environment markers with a PEP 440 version as value such as `python_version` /// Those environment markers with a PEP 440 version as value such as `python_version`
@ -60,6 +60,8 @@ pub enum CanonicalMarkerValueString {
PlatformVersion, PlatformVersion,
/// `implementation_name` /// `implementation_name`
ImplementationName, ImplementationName,
/// `variant_label`
VariantLabel,
} }
impl CanonicalMarkerValueString { impl CanonicalMarkerValueString {
@ -92,6 +94,7 @@ impl From<MarkerValueString> for CanonicalMarkerValueString {
MarkerValueString::PlatformVersionDeprecated => Self::PlatformVersion, MarkerValueString::PlatformVersionDeprecated => Self::PlatformVersion,
MarkerValueString::SysPlatform => Self::SysPlatform, MarkerValueString::SysPlatform => Self::SysPlatform,
MarkerValueString::SysPlatformDeprecated => Self::SysPlatform, MarkerValueString::SysPlatformDeprecated => Self::SysPlatform,
MarkerValueString::VariantLabel => Self::VariantLabel,
} }
} }
} }
@ -109,6 +112,7 @@ impl From<CanonicalMarkerValueString> for MarkerValueString {
CanonicalMarkerValueString::PlatformSystem => Self::PlatformSystem, CanonicalMarkerValueString::PlatformSystem => Self::PlatformSystem,
CanonicalMarkerValueString::PlatformVersion => Self::PlatformVersion, CanonicalMarkerValueString::PlatformVersion => Self::PlatformVersion,
CanonicalMarkerValueString::SysPlatform => Self::SysPlatform, CanonicalMarkerValueString::SysPlatform => Self::SysPlatform,
CanonicalMarkerValueString::VariantLabel => Self::VariantLabel,
} }
} }
} }
@ -125,6 +129,7 @@ impl Display for CanonicalMarkerValueString {
Self::PlatformSystem => f.write_str("platform_system"), Self::PlatformSystem => f.write_str("platform_system"),
Self::PlatformVersion => f.write_str("platform_version"), Self::PlatformVersion => f.write_str("platform_version"),
Self::SysPlatform => f.write_str("sys_platform"), Self::SysPlatform => f.write_str("sys_platform"),
Self::VariantLabel => f.write_str("variant_label"),
} }
} }
} }
@ -163,13 +168,34 @@ impl Display for CanonicalMarkerValueExtra {
/// A key-value pair for `<value> in <key>` or `<value> not in <key>`, where the key is a list. /// A key-value pair for `<value> in <key>` or `<value> not in <key>`, where the key is a list.
/// ///
/// Used for PEP 751 markers. /// Used for PEP 751 and variant markers.
#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] #[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
pub enum CanonicalMarkerListPair { pub enum CanonicalMarkerListPair {
/// A valid [`ExtraName`]. /// A valid [`ExtraName`].
Extras(ExtraName), Extras(ExtraName),
/// A valid [`GroupName`]. /// A valid [`GroupName`].
DependencyGroup(GroupName), DependencyGroup(GroupName),
/// A valid `variant_namespaces`.
VariantNamespaces {
/// If set, the variant marker is evaluated as a variant of the base package.
base: Option<String>,
namespace: VariantNamespace,
},
/// A valid `variant_features`.
VariantFeatures {
/// If set, the variant marker is evaluated as a variant of the base package.
base: Option<String>,
namespace: VariantNamespace,
feature: VariantFeature,
},
/// A valid `variant_properties`.
VariantProperties {
/// If set, the variant marker is evaluated as a variant of the base package.
base: Option<String>,
namespace: VariantNamespace,
feature: VariantFeature,
value: VariantValue,
},
/// For leniency, preserve invalid values. /// For leniency, preserve invalid values.
Arbitrary { key: MarkerValueList, value: String }, Arbitrary { key: MarkerValueList, value: String },
} }
@ -180,6 +206,9 @@ impl CanonicalMarkerListPair {
match self { match self {
Self::Extras(_) => MarkerValueList::Extras, Self::Extras(_) => MarkerValueList::Extras,
Self::DependencyGroup(_) => MarkerValueList::DependencyGroups, Self::DependencyGroup(_) => MarkerValueList::DependencyGroups,
Self::VariantNamespaces { .. } => MarkerValueList::VariantNamespaces,
Self::VariantFeatures { .. } => MarkerValueList::VariantFeatures,
Self::VariantProperties { .. } => MarkerValueList::VariantProperties,
Self::Arbitrary { key, .. } => *key, Self::Arbitrary { key, .. } => *key,
} }
} }
@ -189,6 +218,39 @@ impl CanonicalMarkerListPair {
match self { match self {
Self::Extras(extra) => extra.to_string(), Self::Extras(extra) => extra.to_string(),
Self::DependencyGroup(group) => group.to_string(), Self::DependencyGroup(group) => group.to_string(),
Self::VariantNamespaces {
base: prefix,
namespace,
} => {
if let Some(prefix) = prefix {
format!("{prefix} | {namespace}")
} else {
namespace.to_string()
}
}
Self::VariantFeatures {
base: prefix,
namespace,
feature,
} => {
if let Some(prefix) = prefix {
format!("{prefix} | {namespace} :: {feature}")
} else {
format!("{namespace} :: {feature}")
}
}
Self::VariantProperties {
base: prefix,
namespace,
feature,
value,
} => {
if let Some(prefix) = prefix {
format!("{prefix} | {namespace} :: {feature} :: {value}")
} else {
format!("{namespace} :: {feature} :: {value}")
}
}
Self::Arbitrary { value, .. } => value.clone(), Self::Arbitrary { value, .. } => value.clone(),
} }
} }

View File

@ -15,6 +15,7 @@ mod lowering;
pub(crate) mod parse; pub(crate) mod parse;
mod simplify; mod simplify;
mod tree; mod tree;
mod variants;
pub use environment::{MarkerEnvironment, MarkerEnvironmentBuilder}; pub use environment::{MarkerEnvironment, MarkerEnvironmentBuilder};
pub use lowering::{ pub use lowering::{
@ -24,8 +25,10 @@ pub use tree::{
ContainsMarkerTree, ExtraMarkerTree, ExtraOperator, InMarkerTree, MarkerExpression, ContainsMarkerTree, ExtraMarkerTree, ExtraOperator, InMarkerTree, MarkerExpression,
MarkerOperator, MarkerTree, MarkerTreeContents, MarkerTreeDebugGraph, MarkerTreeKind, MarkerOperator, MarkerTree, MarkerTreeContents, MarkerTreeDebugGraph, MarkerTreeKind,
MarkerValue, MarkerValueExtra, MarkerValueList, MarkerValueString, MarkerValueVersion, MarkerValue, MarkerValueExtra, MarkerValueList, MarkerValueString, MarkerValueVersion,
MarkerWarningKind, StringMarkerTree, StringVersion, VersionMarkerTree, MarkerVariantsEnvironment, MarkerVariantsUniversal, MarkerWarningKind, StringMarkerTree,
StringVersion, VersionMarkerTree,
}; };
pub use variants::{VariantFeature, VariantNamespace, VariantParseError, VariantValue};
/// `serde` helpers for [`MarkerTree`]. /// `serde` helpers for [`MarkerTree`].
pub mod ser { pub mod ser {

View File

@ -4,9 +4,9 @@ use uv_normalize::{ExtraName, GroupName};
use uv_pep440::{Version, VersionPattern, VersionSpecifier}; use uv_pep440::{Version, VersionPattern, VersionSpecifier};
use crate::cursor::Cursor; use crate::cursor::Cursor;
use crate::marker::MarkerValueExtra;
use crate::marker::lowering::CanonicalMarkerListPair; use crate::marker::lowering::CanonicalMarkerListPair;
use crate::marker::tree::{ContainerOperator, MarkerValueList}; use crate::marker::tree::{ContainerOperator, MarkerValueList};
use crate::marker::{MarkerValueExtra, VariantFeature, VariantNamespace, VariantValue};
use crate::{ use crate::{
ExtraOperator, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue, MarkerValueString, ExtraOperator, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue, MarkerValueString,
MarkerValueVersion, MarkerWarningKind, Pep508Error, Pep508ErrorSource, Pep508Url, Reporter, MarkerValueVersion, MarkerWarningKind, Pep508Error, Pep508ErrorSource, Pep508Url, Reporter,
@ -181,6 +181,9 @@ pub(crate) fn parse_marker_key_op_value<T: Pep508Url>(
let r_value = parse_marker_value(cursor, reporter)?; let r_value = parse_marker_value(cursor, reporter)?;
let len = cursor.pos() - start; let len = cursor.pos() - start;
// TODO(konsti): Catch incorrect variant markers in all places, now that we have the
// opportunity to check from the beginning.
// Convert a `<marker_value> <marker_op> <marker_value>` expression into its // Convert a `<marker_value> <marker_op> <marker_value>` expression into its
// typed equivalent. // typed equivalent.
let expr = match l_value { let expr = match l_value {
@ -307,7 +310,7 @@ pub(crate) fn parse_marker_key_op_value<T: Pep508Url>(
Ok(name) => CanonicalMarkerListPair::Extras(name), Ok(name) => CanonicalMarkerListPair::Extras(name),
Err(err) => { Err(err) => {
reporter.report( reporter.report(
MarkerWarningKind::ExtrasInvalidComparison, MarkerWarningKind::ListInvalidComparison,
format!("Expected extra name (found `{l_string}`): {err}"), format!("Expected extra name (found `{l_string}`): {err}"),
); );
CanonicalMarkerListPair::Arbitrary { CanonicalMarkerListPair::Arbitrary {
@ -322,7 +325,7 @@ pub(crate) fn parse_marker_key_op_value<T: Pep508Url>(
Ok(name) => CanonicalMarkerListPair::DependencyGroup(name), Ok(name) => CanonicalMarkerListPair::DependencyGroup(name),
Err(err) => { Err(err) => {
reporter.report( reporter.report(
MarkerWarningKind::ExtrasInvalidComparison, MarkerWarningKind::ListInvalidComparison,
format!("Expected dependency group name (found `{l_string}`): {err}"), format!("Expected dependency group name (found `{l_string}`): {err}"),
); );
CanonicalMarkerListPair::Arbitrary { CanonicalMarkerListPair::Arbitrary {
@ -332,6 +335,118 @@ pub(crate) fn parse_marker_key_op_value<T: Pep508Url>(
} }
} }
} }
MarkerValueList::VariantNamespaces => {
let (base, value) =
if let Some((base, value)) = l_string.split_once(" | ") {
(Some(base.trim().to_string()), value)
} else {
(None, l_string.as_str())
};
CanonicalMarkerListPair::VariantNamespaces {
base,
namespace: VariantNamespace::from_str(value).map_err(|err| {
Pep508Error {
message: Pep508ErrorSource::InvalidVariantSegment(err),
start,
len,
input: cursor.to_string(),
}
})?,
}
}
MarkerValueList::VariantFeatures => {
let (base, value) =
if let Some((base, value)) = l_string.split_once(" | ") {
(Some(base.trim().to_string()), value)
} else {
(None, l_string.as_str())
};
if let Some((namespace, feature)) = value.split_once("::") {
CanonicalMarkerListPair::VariantFeatures {
base,
namespace: VariantNamespace::from_str(namespace).map_err(
|err| Pep508Error {
message: Pep508ErrorSource::InvalidVariantSegment(err),
start,
len,
input: cursor.to_string(),
},
)?,
feature: VariantFeature::from_str(feature).map_err(|err| {
Pep508Error {
message: Pep508ErrorSource::InvalidVariantSegment(err),
start,
len,
input: cursor.to_string(),
}
})?,
}
} else {
reporter.report(
MarkerWarningKind::ListInvalidComparison,
format!("Expected variant feature with two components separated by `::`, found `{value}`"),
);
CanonicalMarkerListPair::Arbitrary {
key,
value: value.to_string(),
}
}
}
MarkerValueList::VariantProperties => {
let (base, value) =
if let Some((base, value)) = l_string.trim().split_once(" | ") {
(Some(base.trim().to_string()), value)
} else {
(None, l_string.as_str())
};
let mut components = value.split("::");
if let (Some(namespace), Some(feature), Some(property), None) = (
components.next(),
components.next(),
components.next(),
components.next(),
) {
CanonicalMarkerListPair::VariantProperties {
base,
namespace: VariantNamespace::from_str(namespace).map_err(
|err| Pep508Error {
message: Pep508ErrorSource::InvalidVariantSegment(err),
start,
len,
input: cursor.to_string(),
},
)?,
feature: VariantFeature::from_str(feature).map_err(|err| {
Pep508Error {
message: Pep508ErrorSource::InvalidVariantSegment(err),
start,
len,
input: cursor.to_string(),
}
})?,
value: VariantValue::from_str(property).map_err(|err| {
Pep508Error {
message: Pep508ErrorSource::InvalidVariantSegment(err),
start,
len,
input: cursor.to_string(),
}
})?,
}
} else {
reporter.report(
MarkerWarningKind::ListInvalidComparison,
format!("Expected variant property with three components separated by `::`, found `{value}`"),
);
CanonicalMarkerListPair::Arbitrary {
key,
value: value.to_string(),
}
}
}
}; };
Some(MarkerExpression::List { pair, operator }) Some(MarkerExpression::List { pair, operator })

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,178 @@
use std::fmt::Display;
use std::str::FromStr;
use serde::{Deserialize, Deserializer};
use thiserror::Error;
// The parser and errors are also used for parsing JSON files, so we're keeping the error message
// generic without references to markers.
/// A segment of a variant uses invalid characters.
#[derive(Error, Debug)]
pub enum VariantParseError {
/// The namespace segment of a variant failed to parse.
#[error(
"Invalid character `{invalid}` in variant namespace, only [a-z0-9_] are allowed: {input}"
)]
Namespace {
/// The character outside the allowed character range.
invalid: char,
/// The invalid input string.
input: String,
},
#[error(
"Invalid character `{invalid}` in variant feature, only [a-z0-9_] are allowed: {input}"
)]
/// The feature segment of a variant failed to parse.
Feature {
/// The character outside the allowed character range.
invalid: char,
/// The invalid input string.
input: String,
},
#[error(
"Invalid character `{invalid}` in variant value, only [a-z0-9_.,!>~<=] are allowed: {input}"
)]
/// The value segment of a variant failed to parse.
Value {
/// The character outside the allowed character range.
invalid: char,
/// The invalid input string.
input: String,
},
}
/// The namespace segment in a variant.
///
/// Variant properties have the structure `namespace :: feature ::value`.
///
/// The segment is canonicalized by trimming it.
#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
#[cfg_attr(feature = "serde", serde(transparent))]
pub struct VariantNamespace(String);
impl FromStr for VariantNamespace {
type Err = VariantParseError;
fn from_str(input: &str) -> Result<Self, Self::Err> {
let input = input.trim();
if let Some(invalid) = input
.chars()
.find(|c| !(c.is_ascii_lowercase() || c.is_ascii_digit() || *c == '_'))
{
return Err(VariantParseError::Namespace {
invalid,
input: input.to_string(),
});
}
Ok(Self(input.to_string()))
}
}
impl<'de> Deserialize<'de> for VariantNamespace {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
Self::from_str(&s).map_err(serde::de::Error::custom)
}
}
impl Display for VariantNamespace {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
Display::fmt(&self.0, f)
}
}
/// The feature segment in a variant.
///
/// Variant properties have the structure `namespace :: feature ::value`.
///
/// The segment is canonicalized by trimming it.
#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
#[cfg_attr(feature = "serde", serde(transparent))]
pub struct VariantFeature(String);
impl FromStr for VariantFeature {
type Err = VariantParseError;
fn from_str(input: &str) -> Result<Self, Self::Err> {
let input = input.trim();
if let Some(invalid) = input
.chars()
.find(|c| !(c.is_ascii_lowercase() || c.is_ascii_digit() || *c == '_'))
{
return Err(VariantParseError::Feature {
invalid,
input: input.to_string(),
});
}
Ok(Self(input.to_string()))
}
}
impl<'de> Deserialize<'de> for VariantFeature {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
Self::from_str(&s).map_err(serde::de::Error::custom)
}
}
impl Display for VariantFeature {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
Display::fmt(&self.0, f)
}
}
/// The value segment in a variant.
///
/// Variant properties have the structure `namespace :: feature ::value`.
///
/// The segment is canonicalized by trimming it.
#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
#[cfg_attr(feature = "serde", serde(transparent))]
pub struct VariantValue(String);
impl FromStr for VariantValue {
type Err = VariantParseError;
fn from_str(input: &str) -> Result<Self, Self::Err> {
let input = input.trim();
if let Some(invalid) = input.chars().find(|c| {
!(c.is_ascii_lowercase()
|| c.is_ascii_digit()
|| matches!(*c, '_' | '.' | ',' | '!' | '>' | '~' | '<' | '='))
}) {
return Err(VariantParseError::Value {
invalid,
input: input.to_string(),
});
}
Ok(Self(input.to_string()))
}
}
impl<'de> Deserialize<'de> for VariantValue {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
Self::from_str(&s).map_err(serde::de::Error::custom)
}
}
impl Display for VariantValue {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
Display::fmt(&self.0, f)
}
}

View File

@ -8,9 +8,10 @@ use uv_normalize::ExtraName;
use crate::marker::parse; use crate::marker::parse;
use crate::{ use crate::{
Cursor, MarkerEnvironment, MarkerTree, Pep508Error, Pep508ErrorSource, Pep508Url, Reporter, Cursor, MarkerEnvironment, MarkerTree, MarkerVariantsEnvironment, Pep508Error,
RequirementOrigin, Scheme, TracingReporter, VerbatimUrl, VerbatimUrlError, expand_env_vars, Pep508ErrorSource, Pep508Url, Reporter, RequirementOrigin, Scheme, TracingReporter,
parse_extras_cursor, split_extras, split_scheme, strip_host, VerbatimUrl, VerbatimUrlError, expand_env_vars, parse_extras_cursor, split_extras,
split_scheme, strip_host,
}; };
/// An extension over [`Pep508Url`] that also supports parsing unnamed requirements, namely paths. /// An extension over [`Pep508Url`] that also supports parsing unnamed requirements, namely paths.
@ -82,17 +83,24 @@ pub struct UnnamedRequirement<ReqUrl: UnnamedRequirementUrl = VerbatimUrl> {
impl<Url: UnnamedRequirementUrl> UnnamedRequirement<Url> { impl<Url: UnnamedRequirementUrl> UnnamedRequirement<Url> {
/// Returns whether the markers apply for the given environment /// Returns whether the markers apply for the given environment
pub fn evaluate_markers(&self, env: &MarkerEnvironment, extras: &[ExtraName]) -> bool { pub fn evaluate_markers(
self.evaluate_optional_environment(Some(env), extras) &self,
env: &MarkerEnvironment,
variants: &impl MarkerVariantsEnvironment,
extras: &[ExtraName],
) -> bool {
self.evaluate_optional_environment(Some(env), variants, extras)
} }
/// Returns whether the markers apply for the given environment /// Returns whether the markers apply for the given environment
pub fn evaluate_optional_environment( pub fn evaluate_optional_environment(
&self, &self,
env: Option<&MarkerEnvironment>, env: Option<&MarkerEnvironment>,
variants: &impl MarkerVariantsEnvironment,
extras: &[ExtraName], extras: &[ExtraName],
) -> bool { ) -> bool {
self.marker.evaluate_optional_environment(env, extras) self.marker
.evaluate_optional_environment(env, variants, extras)
} }
/// Set the source file containing the requirement. /// Set the source file containing the requirement.

View File

@ -27,6 +27,7 @@ uv-small-str = { workspace = true }
hashbrown = { workspace = true } hashbrown = { workspace = true }
indexmap = { workspace = true, features = ["serde"] } indexmap = { workspace = true, features = ["serde"] }
indoc = { workspace = true }
itertools = { workspace = true } itertools = { workspace = true }
jiff = { workspace = true, features = ["serde"] } jiff = { workspace = true, features = ["serde"] }
mailparse = { workspace = true } mailparse = { workspace = true }

View File

@ -10,6 +10,7 @@ pub use parsed_url::*;
pub use scheme::*; pub use scheme::*;
pub use simple_json::*; pub use simple_json::*;
pub use supported_environments::*; pub use supported_environments::*;
pub use variants::*;
mod base_url; mod base_url;
mod conflicts; mod conflicts;
@ -23,3 +24,4 @@ mod parsed_url;
mod scheme; mod scheme;
mod simple_json; mod simple_json;
mod supported_environments; mod supported_environments;
mod variants;

View File

@ -0,0 +1,31 @@
use indoc::formatdoc;
use crate::VerbatimParsedUrl;
#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct VariantProviderBackend {
/// The provider backend string such as `fictional_tech.provider`.
pub backend: String,
/// The requirements that the backend requires (e.g., `["fictional_tech>=1.0"]`).
pub requires: Vec<uv_pep508::Requirement<VerbatimParsedUrl>>,
}
impl VariantProviderBackend {
pub fn import(&self) -> String {
let import = if let Some((path, object)) = self.backend.split_once(':') {
format!("from {path} import {object} as backend")
} else {
format!("import {} as backend", self.backend)
};
formatdoc! {r#"
import sys
if sys.path[0] == "":
sys.path.pop(0)
{import}
"#}
}
}

View File

@ -8,6 +8,7 @@ use tracing::trace;
use uv_configuration::{Constraints, Overrides}; use uv_configuration::{Constraints, Overrides};
use uv_distribution::{DistributionDatabase, Reporter}; use uv_distribution::{DistributionDatabase, Reporter};
use uv_distribution_types::{Dist, DistributionMetadata, Requirement, RequirementSource}; use uv_distribution_types::{Dist, DistributionMetadata, Requirement, RequirementSource};
use uv_pep508::MarkerVariantsUniversal;
use uv_resolver::{InMemoryIndex, MetadataResponse, ResolverEnvironment}; use uv_resolver::{InMemoryIndex, MetadataResponse, ResolverEnvironment};
use uv_types::{BuildContext, HashStrategy, RequestedRequirements}; use uv_types::{BuildContext, HashStrategy, RequestedRequirements};
@ -91,7 +92,13 @@ impl<'a, Context: BuildContext> LookaheadResolver<'a, Context> {
let mut queue: VecDeque<_> = self let mut queue: VecDeque<_> = self
.constraints .constraints
.apply(self.overrides.apply(self.requirements)) .apply(self.overrides.apply(self.requirements))
.filter(|requirement| requirement.evaluate_markers(env.marker_environment(), &[])) .filter(|requirement| {
requirement.evaluate_markers(
env.marker_environment(),
&MarkerVariantsUniversal,
&[],
)
})
.map(|requirement| (*requirement).clone()) .map(|requirement| (*requirement).clone())
.collect(); .collect();
@ -110,9 +117,11 @@ impl<'a, Context: BuildContext> LookaheadResolver<'a, Context> {
.constraints .constraints
.apply(self.overrides.apply(lookahead.requirements())) .apply(self.overrides.apply(lookahead.requirements()))
{ {
if requirement if requirement.evaluate_markers(
.evaluate_markers(env.marker_environment(), lookahead.extras()) env.marker_environment(),
{ &MarkerVariantsUniversal,
lookahead.extras(),
) {
queue.push_back((*requirement).clone()); queue.push_back((*requirement).clone());
} }
} }

View File

@ -42,6 +42,7 @@ uv-small-str = { workspace = true }
uv-static = { workspace = true } uv-static = { workspace = true }
uv-torch = { workspace = true } uv-torch = { workspace = true }
uv-types = { workspace = true } uv-types = { workspace = true }
uv-variants = { workspace = true }
uv-version = { workspace = true } uv-version = { workspace = true }
uv-warnings = { workspace = true } uv-warnings = { workspace = true }
uv-workspace = { workspace = true } uv-workspace = { workspace = true }

View File

@ -1,9 +1,8 @@
use std::fmt::{Display, Formatter};
use either::Either; use either::Either;
use itertools::Itertools; use itertools::Itertools;
use pubgrub::Range; use pubgrub::Range;
use smallvec::SmallVec; use smallvec::SmallVec;
use std::fmt::{Display, Formatter};
use tracing::{debug, trace}; use tracing::{debug, trace};
use uv_configuration::IndexStrategy; use uv_configuration::IndexStrategy;
@ -653,9 +652,9 @@ impl CandidateDist<'_> {
} }
} }
impl<'a> From<&'a PrioritizedDist> for CandidateDist<'a> { impl<'a> CandidateDist<'a> {
fn from(value: &'a PrioritizedDist) -> Self { fn from_prioritized_dist(value: &'a PrioritizedDist, allow_all_variants: bool) -> Self {
if let Some(dist) = value.get() { if let Some(dist) = value.get(allow_all_variants) {
CandidateDist::Compatible(dist) CandidateDist::Compatible(dist)
} else { } else {
// TODO(zanieb) // TODO(zanieb)
@ -664,7 +663,7 @@ impl<'a> From<&'a PrioritizedDist> for CandidateDist<'a> {
// why neither distribution kind can be used. // why neither distribution kind can be used.
let dist = if let Some(incompatibility) = value.incompatible_source() { let dist = if let Some(incompatibility) = value.incompatible_source() {
IncompatibleDist::Source(incompatibility.clone()) IncompatibleDist::Source(incompatibility.clone())
} else if let Some(incompatibility) = value.incompatible_wheel() { } else if let Some(incompatibility) = value.incompatible_wheel(allow_all_variants) {
IncompatibleDist::Wheel(incompatibility.clone()) IncompatibleDist::Wheel(incompatibility.clone())
} else { } else {
IncompatibleDist::Unavailable IncompatibleDist::Unavailable
@ -721,11 +720,42 @@ impl<'a> Candidate<'a> {
Self { Self {
name, name,
version, version,
dist: CandidateDist::from(dist), dist: CandidateDist::from_prioritized_dist(dist, false),
choice_kind, choice_kind,
} }
} }
/// By default, variant wheels are considered incompatible. During universal resolutions,
/// variant wheels should be allowed, similar to any other wheel that is only tag-incompatible
/// to the current platform.
pub(crate) fn allow_variant_wheels(self) -> Self {
// Optimization: Only if the current candidate is incompatible for being a variant, it can
// change if we allow variants.
let CandidateDist::Incompatible {
incompatible_dist: IncompatibleDist::Wheel(IncompatibleWheel::Variant),
prioritized_dist,
} = self.dist
else {
return self;
};
Self {
dist: CandidateDist::from_prioritized_dist(prioritized_dist, true),
..self
}
}
// TODO(konsti): Stop breaking isolation?
pub(crate) fn prioritize_best_variant_wheel(
self,
prioritized_dist: &'a PrioritizedDist,
) -> Self {
Self {
dist: CandidateDist::from_prioritized_dist(prioritized_dist, true),
..self
}
}
/// Return the name of the package. /// Return the name of the package.
pub(crate) fn name(&self) -> &PackageName { pub(crate) fn name(&self) -> &PackageName {
self.name self.name

View File

@ -16,7 +16,9 @@ use uv_distribution_types::{
}; };
use uv_normalize::{ExtraName, InvalidNameError, PackageName}; use uv_normalize::{ExtraName, InvalidNameError, PackageName};
use uv_pep440::{LocalVersionSlice, LowerBound, Version, VersionSpecifier}; use uv_pep440::{LocalVersionSlice, LowerBound, Version, VersionSpecifier};
use uv_pep508::{MarkerEnvironment, MarkerExpression, MarkerTree, MarkerValueVersion}; use uv_pep508::{
MarkerEnvironment, MarkerExpression, MarkerTree, MarkerValueVersion, MarkerVariantsUniversal,
};
use uv_platform_tags::Tags; use uv_platform_tags::Tags;
use uv_pypi_types::ParsedUrl; use uv_pypi_types::ParsedUrl;
use uv_redacted::DisplaySafeUrl; use uv_redacted::DisplaySafeUrl;
@ -45,6 +47,9 @@ pub enum ResolveError {
DerivationChain, DerivationChain,
), ),
#[error(transparent)]
VariantFrontend(uv_distribution::Error),
#[error(transparent)] #[error(transparent)]
Client(#[from] uv_client::Error), Client(#[from] uv_client::Error),
@ -409,7 +414,7 @@ impl NoSolutionError {
":".bold(), ":".bold(),
current_python_version, current_python_version,
)?; )?;
} else if !markers.evaluate(&self.current_environment, &[]) { } else if !markers.evaluate(&self.current_environment, &MarkerVariantsUniversal, &[]) {
write!( write!(
f, f,
"\n\n{}{} The resolution failed for an environment that is not the current one, \ "\n\n{}{} The resolution failed for an environment that is not the current one, \

View File

@ -1,22 +1,23 @@
use std::collections::BTreeMap;
use std::collections::btree_map::Entry;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use tracing::instrument; use tracing::instrument;
use std::collections::BTreeMap;
use std::collections::btree_map::Entry;
use uv_client::{FlatIndexEntries, FlatIndexEntry}; use uv_client::{FlatIndexEntries, FlatIndexEntry};
use uv_configuration::BuildOptions; use uv_configuration::BuildOptions;
use uv_distribution_filename::{DistFilename, SourceDistFilename, WheelFilename}; use uv_distribution_filename::{DistFilename, SourceDistFilename, WheelFilename};
use uv_distribution_types::{ use uv_distribution_types::{
File, HashComparison, HashPolicy, IncompatibleSource, IncompatibleWheel, IndexUrl, File, HashComparison, HashPolicy, IncompatibleSource, IncompatibleWheel, IndexEntryFilename,
PrioritizedDist, RegistryBuiltWheel, RegistrySourceDist, SourceDistCompatibility, IndexUrl, PrioritizedDist, RegistryBuiltWheel, RegistrySourceDist, RegistryVariantsJson,
WheelCompatibility, SourceDistCompatibility, WheelCompatibility,
}; };
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_pep440::Version; use uv_pep440::Version;
use uv_platform_tags::{TagCompatibility, Tags}; use uv_platform_tags::{TagCompatibility, Tags};
use uv_pypi_types::HashDigest; use uv_pypi_types::HashDigest;
use uv_types::HashStrategy; use uv_types::HashStrategy;
use uv_variants::VariantPriority;
/// A set of [`PrioritizedDist`] from a `--find-links` entry, indexed by [`PackageName`] /// A set of [`PrioritizedDist`] from a `--find-links` entry, indexed by [`PackageName`]
/// and [`Version`]. /// and [`Version`].
@ -112,7 +113,7 @@ impl FlatDistributions {
fn add_file( fn add_file(
&mut self, &mut self,
file: File, file: File,
filename: DistFilename, filename: IndexEntryFilename,
tags: Option<&Tags>, tags: Option<&Tags>,
hasher: &HashStrategy, hasher: &HashStrategy,
build_options: &BuildOptions, build_options: &BuildOptions,
@ -121,7 +122,7 @@ impl FlatDistributions {
// No `requires-python` here: for source distributions, we don't have that information; // No `requires-python` here: for source distributions, we don't have that information;
// for wheels, we read it lazily only when selected. // for wheels, we read it lazily only when selected.
match filename { match filename {
DistFilename::WheelFilename(filename) => { IndexEntryFilename::DistFilename(DistFilename::WheelFilename(filename)) => {
let version = filename.version.clone(); let version = filename.version.clone();
let compatibility = Self::wheel_compatibility( let compatibility = Self::wheel_compatibility(
@ -145,7 +146,7 @@ impl FlatDistributions {
} }
} }
} }
DistFilename::SourceDistFilename(filename) => { IndexEntryFilename::DistFilename(DistFilename::SourceDistFilename(filename)) => {
let compatibility = Self::source_dist_compatibility( let compatibility = Self::source_dist_compatibility(
&filename, &filename,
file.hashes.as_slice(), file.hashes.as_slice(),
@ -169,6 +170,22 @@ impl FlatDistributions {
} }
} }
} }
IndexEntryFilename::VariantJson(variants_json) => {
let version = variants_json.version.clone();
let registry_variants_json = RegistryVariantsJson {
filename: variants_json,
file: Box::new(file),
index,
};
match self.0.entry(version) {
Entry::Occupied(mut entry) => {
entry.get_mut().insert_variant_json(registry_variants_json);
}
Entry::Vacant(entry) => {
entry.insert(PrioritizedDist::from_variant_json(registry_variants_json));
}
}
}
} }
} }
@ -214,7 +231,7 @@ impl FlatDistributions {
} }
// Determine a compatibility for the wheel based on tags. // Determine a compatibility for the wheel based on tags.
let priority = match tags { let tag_priority = match tags {
Some(tags) => match filename.compatibility(tags) { Some(tags) => match filename.compatibility(tags) {
TagCompatibility::Incompatible(tag) => { TagCompatibility::Incompatible(tag) => {
return WheelCompatibility::Incompatible(IncompatibleWheel::Tag(tag)); return WheelCompatibility::Incompatible(IncompatibleWheel::Tag(tag));
@ -224,6 +241,13 @@ impl FlatDistributions {
None => None, None => None,
}; };
// TODO(konsti): Currently we ignore variants here on only determine them later
let variant_priority = if filename.variant().is_none() {
VariantPriority::NonVariant
} else {
VariantPriority::Unknown
};
// Check if hashes line up. // Check if hashes line up.
let hash = if let HashPolicy::Validate(required) = let hash = if let HashPolicy::Validate(required) =
hasher.get_package(&filename.name, &filename.version) hasher.get_package(&filename.name, &filename.version)
@ -242,7 +266,12 @@ impl FlatDistributions {
// Break ties with the build tag. // Break ties with the build tag.
let build_tag = filename.build_tag().cloned(); let build_tag = filename.build_tag().cloned();
WheelCompatibility::Compatible(hash, priority, build_tag) WheelCompatibility::Compatible {
hash,
variant_priority,
tag_priority,
build_tag,
}
} }
} }

View File

@ -254,7 +254,6 @@ pub(crate) fn simplify_conflict_markers(
// For example, when `a` and `b` conflict, this marker does not simplify: // For example, when `a` and `b` conflict, this marker does not simplify:
// ``` // ```
// (platform_machine == 'x86_64' and extra == 'extra-5-foo-b') or extra == 'extra-5-foo-a' // (platform_machine == 'x86_64' and extra == 'extra-5-foo-b') or extra == 'extra-5-foo-a'
// ````
graph[edge_index].evaluate_only_extras(&extras, &groups) graph[edge_index].evaluate_only_extras(&extras, &groups)
}); });
if all_paths_satisfied { if all_paths_satisfied {

View File

@ -23,7 +23,7 @@ pub use resolution_mode::ResolutionMode;
pub use resolver::{ pub use resolver::{
BuildId, DefaultResolverProvider, DerivationChainBuilder, InMemoryIndex, MetadataResponse, BuildId, DefaultResolverProvider, DerivationChainBuilder, InMemoryIndex, MetadataResponse,
PackageVersionsResult, Reporter as ResolverReporter, Resolver, ResolverEnvironment, PackageVersionsResult, Reporter as ResolverReporter, Resolver, ResolverEnvironment,
ResolverProvider, VersionsResponse, WheelMetadataResult, ResolverProvider, VariantProviderResult, VersionsResponse, WheelMetadataResult,
}; };
pub use universal_marker::{ConflictMarker, UniversalMarker}; pub use universal_marker::{ConflictMarker, UniversalMarker};
pub use version_map::VersionMap; pub use version_map::VersionMap;

View File

@ -31,7 +31,7 @@ use uv_git::{RepositoryReference, ResolvedRepositoryReference};
use uv_git_types::{GitLfs, GitOid, GitReference, GitUrl, GitUrlParseError}; use uv_git_types::{GitLfs, GitOid, GitReference, GitUrl, GitUrlParseError};
use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_normalize::{ExtraName, GroupName, PackageName};
use uv_pep440::Version; use uv_pep440::Version;
use uv_pep508::{MarkerEnvironment, MarkerTree, VerbatimUrl}; use uv_pep508::{MarkerEnvironment, MarkerTree, MarkerVariantsUniversal, VerbatimUrl};
use uv_platform_tags::{TagCompatibility, TagPriority, Tags}; use uv_platform_tags::{TagCompatibility, TagPriority, Tags};
use uv_pypi_types::{HashDigests, Hashes, ParsedGitUrl, VcsKind}; use uv_pypi_types::{HashDigests, Hashes, ParsedGitUrl, VcsKind};
use uv_redacted::DisplaySafeUrl; use uv_redacted::DisplaySafeUrl;
@ -365,7 +365,7 @@ impl<'lock> PylockToml {
if !node.is_base() { if !node.is_base() {
continue; continue;
} }
let ResolvedDist::Installable { dist, version } = &node.dist else { let ResolvedDist::Installable { dist, version, .. } = &node.dist else {
continue; continue;
}; };
if omit.contains(dist.name()) { if omit.contains(dist.name()) {
@ -981,7 +981,10 @@ impl<'lock> PylockToml {
for package in self.packages { for package in self.packages {
// Omit packages that aren't relevant to the current environment. // Omit packages that aren't relevant to the current environment.
if !package.marker.evaluate_pep751(markers, extras, groups) { if !package
.marker
.evaluate_pep751(markers, &MarkerVariantsUniversal, extras, groups)
{
continue; continue;
} }
@ -1060,6 +1063,7 @@ impl<'lock> PylockToml {
})); }));
let dist = ResolvedDist::Installable { let dist = ResolvedDist::Installable {
dist: Arc::new(built_dist), dist: Arc::new(built_dist),
variants_json: None,
version: package.version, version: package.version,
}; };
Node::Dist { Node::Dist {
@ -1077,6 +1081,7 @@ impl<'lock> PylockToml {
)?)); )?));
let dist = ResolvedDist::Installable { let dist = ResolvedDist::Installable {
dist: Arc::new(sdist), dist: Arc::new(sdist),
variants_json: None,
version: package.version, version: package.version,
}; };
Node::Dist { Node::Dist {
@ -1091,6 +1096,7 @@ impl<'lock> PylockToml {
)); ));
let dist = ResolvedDist::Installable { let dist = ResolvedDist::Installable {
dist: Arc::new(sdist), dist: Arc::new(sdist),
variants_json: None,
version: package.version, version: package.version,
}; };
Node::Dist { Node::Dist {
@ -1105,6 +1111,7 @@ impl<'lock> PylockToml {
)); ));
let dist = ResolvedDist::Installable { let dist = ResolvedDist::Installable {
dist: Arc::new(sdist), dist: Arc::new(sdist),
variants_json: None,
version: package.version, version: package.version,
}; };
Node::Dist { Node::Dist {
@ -1121,6 +1128,7 @@ impl<'lock> PylockToml {
let dist = dist.to_dist(install_path, &package.name, package.version.as_ref())?; let dist = dist.to_dist(install_path, &package.name, package.version.as_ref())?;
let dist = ResolvedDist::Installable { let dist = ResolvedDist::Installable {
dist: Arc::new(dist), dist: Arc::new(dist),
variants_json: None,
version: package.version, version: package.version,
}; };
Node::Dist { Node::Dist {

View File

@ -5,16 +5,24 @@ use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use either::Either; use either::Either;
use hashbrown::HashMap;
use itertools::Itertools; use itertools::Itertools;
use petgraph::Graph; use petgraph::Graph;
use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet}; use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet};
use uv_configuration::ExtrasSpecificationWithDefaults; use uv_configuration::ExtrasSpecificationWithDefaults;
use uv_configuration::{BuildOptions, DependencyGroupsWithDefaults, InstallOptions}; use uv_configuration::{BuildOptions, DependencyGroupsWithDefaults, InstallOptions};
use uv_distribution::{DistributionDatabase, PackageVariantCache};
use uv_distribution_types::{Edge, Node, Resolution, ResolvedDist}; use uv_distribution_types::{Edge, Node, Resolution, ResolvedDist};
use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_normalize::{ExtraName, GroupName, PackageName};
use uv_pep508::{
MarkerVariantsEnvironment, MarkerVariantsUniversal, VariantFeature, VariantNamespace,
VariantValue,
};
use uv_platform_tags::Tags; use uv_platform_tags::Tags;
use uv_pypi_types::ResolverMarkerEnvironment; use uv_pypi_types::ResolverMarkerEnvironment;
use uv_types::BuildContext;
use uv_variants::variant_with_label::VariantWithLabel;
use crate::lock::{LockErrorKind, Package, TagPolicy}; use crate::lock::{LockErrorKind, Package, TagPolicy};
use crate::{Lock, LockError}; use crate::{Lock, LockError};
@ -33,7 +41,8 @@ pub trait Installable<'lock> {
fn project_name(&self) -> Option<&PackageName>; fn project_name(&self) -> Option<&PackageName>;
/// Convert the [`Lock`] to a [`Resolution`] using the given marker environment, tags, and root. /// Convert the [`Lock`] to a [`Resolution`] using the given marker environment, tags, and root.
fn to_resolution( #[allow(async_fn_in_trait)]
async fn to_resolution<Context: BuildContext>(
&self, &self,
marker_env: &ResolverMarkerEnvironment, marker_env: &ResolverMarkerEnvironment,
tags: &Tags, tags: &Tags,
@ -41,6 +50,8 @@ pub trait Installable<'lock> {
groups: &DependencyGroupsWithDefaults, groups: &DependencyGroupsWithDefaults,
build_options: &BuildOptions, build_options: &BuildOptions,
install_options: &InstallOptions, install_options: &InstallOptions,
distribution_database: DistributionDatabase<'_, Context>,
variants_cache: &PackageVariantCache,
) -> Result<Resolution, LockError> { ) -> Result<Resolution, LockError> {
let size_guess = self.lock().packages.len(); let size_guess = self.lock().packages.len();
let mut petgraph = Graph::with_capacity(size_guess, size_guess); let mut petgraph = Graph::with_capacity(size_guess, size_guess);
@ -52,6 +63,8 @@ pub trait Installable<'lock> {
let mut activated_extras: Vec<(&PackageName, &ExtraName)> = vec![]; let mut activated_extras: Vec<(&PackageName, &ExtraName)> = vec![];
let mut activated_groups: Vec<(&PackageName, &GroupName)> = vec![]; let mut activated_groups: Vec<(&PackageName, &GroupName)> = vec![];
let mut resolved_variants = QueriedVariants::default();
let root = petgraph.add_node(Node::Root); let root = petgraph.add_node(Node::Root);
// Determine the set of activated extras and groups, from the root. // Determine the set of activated extras and groups, from the root.
@ -143,8 +156,10 @@ pub trait Installable<'lock> {
}) })
.flatten() .flatten()
{ {
// TODO(konsti): Evaluate variant declarations on workspace/path dependencies.
if !dep.complexified_marker.evaluate( if !dep.complexified_marker.evaluate(
marker_env, marker_env,
&MarkerVariantsUniversal,
activated_projects.iter().copied(), activated_projects.iter().copied(),
activated_extras.iter().copied(), activated_extras.iter().copied(),
activated_groups.iter().copied(), activated_groups.iter().copied(),
@ -211,7 +226,11 @@ pub trait Installable<'lock> {
// Add any requirements that are exclusive to the workspace root (e.g., dependencies in // Add any requirements that are exclusive to the workspace root (e.g., dependencies in
// PEP 723 scripts). // PEP 723 scripts).
for dependency in self.lock().requirements() { for dependency in self.lock().requirements() {
if !dependency.marker.evaluate(marker_env, &[]) { if !dependency
.marker
// No package, evaluate markers to false.
.evaluate(marker_env, &Vec::new().as_slice(), &[])
{
continue; continue;
} }
@ -263,11 +282,16 @@ pub trait Installable<'lock> {
}) })
.flatten() .flatten()
{ {
if !dependency.marker.evaluate(marker_env, &[]) { // TODO(konsti): Evaluate markers for the current package
if !dependency
.marker
.evaluate(marker_env, &MarkerVariantsUniversal, &[])
{
continue; continue;
} }
let root_name = &dependency.name; let root_name = &dependency.name;
// TODO(konsti): Evaluate variant declarations on workspace/path dependencies.
let dist = self let dist = self
.lock() .lock()
.find_by_markers(root_name, marker_env) .find_by_markers(root_name, marker_env)
@ -377,8 +401,10 @@ pub trait Installable<'lock> {
additional_activated_extras.push(key); additional_activated_extras.push(key);
} }
} }
// TODO(konsti): Evaluate variants
if !dep.complexified_marker.evaluate( if !dep.complexified_marker.evaluate(
marker_env, marker_env,
&MarkerVariantsUniversal,
activated_projects.iter().copied(), activated_projects.iter().copied(),
activated_extras activated_extras
.iter() .iter()
@ -464,9 +490,40 @@ pub trait Installable<'lock> {
} else { } else {
Either::Right(package.dependencies.iter()) Either::Right(package.dependencies.iter())
}; };
let variant_base = format!(
"{}=={}",
package.id.name,
package
.version()
.map(ToString::to_string)
.unwrap_or("TODO(konsti)".to_string())
);
if !resolved_variants
.0
.contains_key(&package.name().to_string())
{
let variant_properties = determine_properties(
package,
self.install_path(),
marker_env,
&distribution_database,
variants_cache,
)
.await?;
resolved_variants
.0
.insert(variant_base.clone(), variant_properties);
}
for dep in deps { for dep in deps {
if !dep.complexified_marker.evaluate( if !dep.complexified_marker.evaluate(
marker_env, marker_env,
&CurrentQueriedVariants {
global: &resolved_variants,
current: resolved_variants.0.get(&variant_base).unwrap(),
},
activated_projects.iter().copied(), activated_projects.iter().copied(),
activated_extras.iter().copied(), activated_extras.iter().copied(),
activated_groups.iter().copied(), activated_groups.iter().copied(),
@ -534,8 +591,10 @@ pub trait Installable<'lock> {
marker_env, marker_env,
)?; )?;
let version = package.version().cloned(); let version = package.version().cloned();
let variants_json = package.to_registry_variants_json(self.install_path())?;
let dist = ResolvedDist::Installable { let dist = ResolvedDist::Installable {
dist: Arc::new(dist), dist: Arc::new(dist),
variants_json: variants_json.map(Arc::new),
version, version,
}; };
let hashes = package.hashes(); let hashes = package.hashes();
@ -562,6 +621,8 @@ pub trait Installable<'lock> {
let version = package.version().cloned(); let version = package.version().cloned();
let dist = ResolvedDist::Installable { let dist = ResolvedDist::Installable {
dist: Arc::new(dist), dist: Arc::new(dist),
// No need to determine variants for something we don't install.
variants_json: None,
version, version,
}; };
let hashes = package.hashes(); let hashes = package.hashes();
@ -592,3 +653,140 @@ pub trait Installable<'lock> {
} }
} }
} }
/// Map for the package identifier to the package's variants for marker evaluation.
#[derive(Default, Debug)]
struct QueriedVariants(HashMap<String, VariantWithLabel>);
/// Variants for markers evaluation both for the current package (without base) and globally (with
/// base).
#[derive(Copy, Clone, Debug)]
struct CurrentQueriedVariants<'a> {
current: &'a VariantWithLabel,
global: &'a QueriedVariants,
}
impl MarkerVariantsEnvironment for CurrentQueriedVariants<'_> {
fn contains_namespace(&self, namespace: &VariantNamespace) -> bool {
self.current.contains_namespace(namespace)
}
fn contains_feature(&self, namespace: &VariantNamespace, feature: &VariantFeature) -> bool {
self.current.contains_feature(namespace, feature)
}
fn contains_property(
&self,
namespace: &VariantNamespace,
feature: &VariantFeature,
value: &VariantValue,
) -> bool {
self.current.contains_property(namespace, feature, value)
}
fn contains_base_namespace(&self, prefix: &str, namespace: &VariantNamespace) -> bool {
let Some(variant) = self.global.0.get(prefix) else {
return false;
};
variant.contains_namespace(namespace)
}
fn contains_base_feature(
&self,
prefix: &str,
namespace: &VariantNamespace,
feature: &VariantFeature,
) -> bool {
let Some(variant) = self.global.0.get(prefix) else {
return false;
};
variant.contains_feature(namespace, feature)
}
fn contains_base_property(
&self,
prefix: &str,
namespace: &VariantNamespace,
feature: &VariantFeature,
value: &VariantValue,
) -> bool {
let Some(variant) = self.global.0.get(prefix) else {
return false;
};
variant.contains_property(namespace, feature, value)
}
fn label(&self) -> Option<&str> {
self.current.label()
}
}
async fn determine_properties<Context: BuildContext>(
package: &Package,
workspace_root: &Path,
marker_env: &ResolverMarkerEnvironment,
distribution_database: &DistributionDatabase<'_, Context>,
variants_cache: &PackageVariantCache,
) -> Result<VariantWithLabel, LockError> {
let Some(variants_json) = package.to_registry_variants_json(workspace_root)? else {
// When selecting a non-variant wheel, all variant markers evaluate to false.
return Ok(VariantWithLabel::default());
};
let resolved_variants = if variants_cache.register(variants_json.version_id()) {
let resolved_variants = distribution_database
.fetch_and_query_variants(&variants_json, marker_env)
.await
.map_err(|err| LockErrorKind::VariantError {
package_id: package.id.clone(),
err,
})?;
let resolved_variants = Arc::new(resolved_variants);
variants_cache.done(variants_json.version_id(), resolved_variants.clone());
resolved_variants
} else {
variants_cache
.wait(&variants_json.version_id())
.await
.expect("missing value for registered task")
};
// Select best wheel
let mut highest_priority_variant_wheel: Option<(_, Vec<usize>)> = None;
for wheel in &package.wheels {
let Some(variant) = wheel.filename.variant() else {
// The non-variant wheel is already supported
continue;
};
let Some(scores) = resolved_variants.score_variant(variant) else {
continue;
};
if let Some((_, old_scores)) = &highest_priority_variant_wheel {
if &scores > old_scores {
highest_priority_variant_wheel = Some((variant, scores));
}
} else {
highest_priority_variant_wheel = Some((variant, scores));
}
}
if let Some((best_variant, _)) = highest_priority_variant_wheel {
// TODO(konsti): We shouldn't need to clone
// TODO(konsti): The variant exists because we used it for scoring, but we should
// be able to write this without unwrap.
let known_properties = resolved_variants.variants_json.variants[best_variant].clone();
Ok(VariantWithLabel {
variant: known_properties,
label: Some(best_variant.clone()),
})
} else {
// When selecting the non-variant wheel, all variant markers evaluate to false.
Ok(VariantWithLabel::default())
}
}

View File

@ -26,10 +26,11 @@ use uv_distribution_filename::{
}; };
use uv_distribution_types::{ use uv_distribution_types::{
BuiltDist, DependencyMetadata, DirectUrlBuiltDist, DirectUrlSourceDist, DirectorySourceDist, BuiltDist, DependencyMetadata, DirectUrlBuiltDist, DirectUrlSourceDist, DirectorySourceDist,
Dist, DistributionMetadata, FileLocation, GitSourceDist, IndexLocations, IndexMetadata, Dist, DistributionMetadata, File, FileLocation, GitSourceDist, IndexLocations, IndexMetadata,
IndexUrl, Name, PathBuiltDist, PathSourceDist, RegistryBuiltDist, RegistryBuiltWheel, IndexUrl, Name, PathBuiltDist, PathSourceDist, RegistryBuiltDist, RegistryBuiltWheel,
RegistrySourceDist, RemoteSource, Requirement, RequirementSource, RequiresPython, ResolvedDist, RegistrySourceDist, RegistryVariantsJson, RemoteSource, Requirement, RequirementSource,
SimplifiedMarkerTree, StaticMetadata, ToUrlError, UrlString, RequiresPython, ResolvedDist, SimplifiedMarkerTree, StaticMetadata, ToUrlError, UrlString,
VariantsJsonFilename,
}; };
use uv_fs::{PortablePath, PortablePathBuf, relative_to}; use uv_fs::{PortablePath, PortablePathBuf, relative_to};
use uv_git::{RepositoryReference, ResolvedRepositoryReference}; use uv_git::{RepositoryReference, ResolvedRepositoryReference};
@ -832,7 +833,10 @@ impl Lock {
&self.manifest.members &self.manifest.members
} }
/// Returns the dependency groups that were used to generate this lock. /// Returns requirements provided to the resolver, exclusive of the workspace members.
///
/// These are requirements that are attached to the project, but not to any of its
/// workspace members. For example, the requirements in a PEP 723 script would be included here.
pub fn requirements(&self) -> &BTreeSet<Requirement> { pub fn requirements(&self) -> &BTreeSet<Requirement> {
&self.manifest.requirements &self.manifest.requirements
} }
@ -2365,6 +2369,10 @@ pub struct Package {
pub(crate) id: PackageId, pub(crate) id: PackageId,
sdist: Option<SourceDist>, sdist: Option<SourceDist>,
wheels: Vec<Wheel>, wheels: Vec<Wheel>,
/// The variants JSON file for the package version, if available.
///
/// Named `variants-json` in `uv.lock`.
variants_json: Option<VariantsJsonEntry>,
/// If there are multiple versions or sources for the same package name, we add the markers of /// If there are multiple versions or sources for the same package name, we add the markers of
/// the fork(s) that contained this version or source, so we can set the correct preferences in /// the fork(s) that contained this version or source, so we can set the correct preferences in
/// the next resolution. /// the next resolution.
@ -2390,6 +2398,7 @@ impl Package {
let id = PackageId::from_annotated_dist(annotated_dist, root)?; let id = PackageId::from_annotated_dist(annotated_dist, root)?;
let sdist = SourceDist::from_annotated_dist(&id, annotated_dist)?; let sdist = SourceDist::from_annotated_dist(&id, annotated_dist)?;
let wheels = Wheel::from_annotated_dist(annotated_dist)?; let wheels = Wheel::from_annotated_dist(annotated_dist)?;
let variants_json = VariantsJsonEntry::from_annotated_dist(annotated_dist)?;
let requires_dist = if id.source.is_immutable() { let requires_dist = if id.source.is_immutable() {
BTreeSet::default() BTreeSet::default()
} else { } else {
@ -2438,6 +2447,7 @@ impl Package {
id, id,
sdist, sdist,
wheels, wheels,
variants_json,
fork_markers, fork_markers,
dependencies: vec![], dependencies: vec![],
optional_dependencies: BTreeMap::default(), optional_dependencies: BTreeMap::default(),
@ -2942,6 +2952,82 @@ impl Package {
Ok(Some(sdist)) Ok(Some(sdist))
} }
/// Convert to a [`RegistryVariantsJson`] for installation.
pub(crate) fn to_registry_variants_json(
&self,
workspace_root: &Path,
) -> Result<Option<RegistryVariantsJson>, LockError> {
let Some(variants_json) = &self.variants_json else {
return Ok(None);
};
let name = &self.id.name;
let version = self
.id
.version
.as_ref()
.expect("version for registry source");
let (file_url, index) = match &self.id.source {
Source::Registry(RegistrySource::Url(url)) => {
let file_url =
variants_json
.url
.url()
.ok_or_else(|| LockErrorKind::MissingUrl {
name: name.clone(),
version: version.clone(),
})?;
let index = IndexUrl::from(VerbatimUrl::from_url(
url.to_url().map_err(LockErrorKind::InvalidUrl)?,
));
(FileLocation::AbsoluteUrl(file_url.clone()), index)
}
Source::Registry(RegistrySource::Path(path)) => {
let index = IndexUrl::from(
VerbatimUrl::from_absolute_path(workspace_root.join(path))
.map_err(LockErrorKind::RegistryVerbatimUrl)?,
);
match &variants_json.url {
VariantsJsonSource::Url { url: file_url } => {
(FileLocation::AbsoluteUrl(file_url.clone()), index)
}
VariantsJsonSource::Path { path: file_path } => {
let file_path = workspace_root.join(path).join(file_path);
let file_url =
DisplaySafeUrl::from_file_path(&file_path).map_err(|()| {
LockErrorKind::PathToUrl {
path: file_path.into_boxed_path(),
}
})?;
(FileLocation::AbsoluteUrl(UrlString::from(file_url)), index)
}
}
}
_ => todo!("Handle error: variants.json can only be used on a registry source"),
};
let filename = format!("{name}-{version}-variants.json");
let file = File {
dist_info_metadata: false,
filename: SmallString::from(filename),
hashes: variants_json.hash.iter().map(|h| h.0.clone()).collect(),
requires_python: None,
size: variants_json.size,
upload_time_utc_ms: variants_json.upload_time.map(Timestamp::as_millisecond),
url: file_url,
yanked: None,
zstd: None,
};
Ok(Some(RegistryVariantsJson {
filename: VariantsJsonFilename {
name: self.name().clone(),
version: version.clone(),
},
file: Box::new(file),
index,
}))
}
fn to_toml( fn to_toml(
&self, &self,
requires_python: &RequiresPython, requires_python: &RequiresPython,
@ -3015,6 +3101,10 @@ impl Package {
table.insert("wheels", value(wheels)); table.insert("wheels", value(wheels));
} }
if let Some(ref variants_json) = self.variants_json {
table.insert("variants-json", value(variants_json.to_toml()?));
}
// Write the package metadata, if non-empty. // Write the package metadata, if non-empty.
{ {
let mut metadata_table = Table::new(); let mut metadata_table = Table::new();
@ -3086,7 +3176,7 @@ impl Package {
} }
fn find_best_wheel(&self, tag_policy: TagPolicy<'_>) -> Option<usize> { fn find_best_wheel(&self, tag_policy: TagPolicy<'_>) -> Option<usize> {
type WheelPriority<'lock> = (TagPriority, Option<&'lock BuildTag>); type WheelPriority<'lock> = (bool, TagPriority, Option<&'lock BuildTag>);
let mut best: Option<(WheelPriority, usize)> = None; let mut best: Option<(WheelPriority, usize)> = None;
for (i, wheel) in self.wheels.iter().enumerate() { for (i, wheel) in self.wheels.iter().enumerate() {
@ -3096,7 +3186,8 @@ impl Package {
continue; continue;
}; };
let build_tag = wheel.filename.build_tag(); let build_tag = wheel.filename.build_tag();
let wheel_priority = (tag_priority, build_tag); // Non-variant wheels before variant wheels.
let wheel_priority = (wheel.filename.variant().is_none(), tag_priority, build_tag);
match best { match best {
None => { None => {
best = Some((wheel_priority, i)); best = Some((wheel_priority, i));
@ -3264,6 +3355,8 @@ struct PackageWire {
sdist: Option<SourceDist>, sdist: Option<SourceDist>,
#[serde(default)] #[serde(default)]
wheels: Vec<Wheel>, wheels: Vec<Wheel>,
#[serde(default, rename = "variants-json")]
variants_json: Option<VariantsJsonEntry>,
#[serde(default, rename = "resolution-markers")] #[serde(default, rename = "resolution-markers")]
fork_markers: Vec<SimplifiedMarkerTree>, fork_markers: Vec<SimplifiedMarkerTree>,
#[serde(default)] #[serde(default)]
@ -3321,6 +3414,7 @@ impl PackageWire {
metadata: self.metadata, metadata: self.metadata,
sdist: self.sdist, sdist: self.sdist,
wheels: self.wheels, wheels: self.wheels,
variants_json: self.variants_json,
fork_markers: self fork_markers: self
.fork_markers .fork_markers
.into_iter() .into_iter()
@ -4405,6 +4499,152 @@ struct ZstdWheel {
size: Option<u64>, size: Option<u64>,
} }
#[derive(Clone, Debug, serde::Deserialize, PartialEq, Eq)]
#[serde(from = "VariantsJsonWire")]
struct VariantsJsonEntry {
/// A URL or file path (via `file://`) where the variants JSON file that was locked
/// against was found. The location does not need to exist in the future,
/// so this should be treated as only a hint to where to look and/or
/// recording where the variants JSON file originally came from.
#[serde(flatten)]
url: VariantsJsonSource,
/// A hash of the variants JSON file.
///
/// This is only present for variants JSON files that come from registries and direct
/// URLs. Files from git or path dependencies do not have hashes
/// associated with them.
hash: Option<Hash>,
/// The size of the variants JSON file in bytes.
///
/// This is only present for variants JSON files that come from registries.
size: Option<u64>,
/// The upload time of the variants JSON file.
///
/// This is only present for variants JSON files that come from registries.
upload_time: Option<Timestamp>,
}
#[derive(Clone, Debug, serde::Deserialize, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
struct VariantsJsonWire {
/// A URL or file path (via `file://`) where the variants JSON file that was locked
/// against was found.
#[serde(flatten)]
url: VariantsJsonSource,
/// A hash of the variants JSON file.
hash: Option<Hash>,
/// The size of the variants JSON file in bytes.
size: Option<u64>,
/// The upload time of the variants JSON file.
#[serde(alias = "upload_time")]
upload_time: Option<Timestamp>,
}
impl VariantsJsonEntry {
fn from_annotated_dist(annotated_dist: &AnnotatedDist) -> Result<Option<Self>, LockError> {
match &annotated_dist.dist {
// We pass empty installed packages for locking.
ResolvedDist::Installed { .. } => unreachable!(),
ResolvedDist::Installable { variants_json, .. } => {
if let Some(variants_json) = variants_json {
let url = match &variants_json.index {
IndexUrl::Pypi(_) | IndexUrl::Url(_) => {
let url = normalize_file_location(&variants_json.file.url)
.map_err(LockErrorKind::InvalidUrl)
.map_err(LockError::from)?;
VariantsJsonSource::Url { url }
}
IndexUrl::Path(path) => {
let index_path = path
.to_file_path()
.map_err(|()| LockErrorKind::UrlToPath { url: path.to_url() })?;
let variants_url = variants_json
.file
.url
.to_url()
.map_err(LockErrorKind::InvalidUrl)?;
if variants_url.scheme() == "file" {
let variants_path = variants_url
.to_file_path()
.map_err(|()| LockErrorKind::UrlToPath { url: variants_url })?;
let path = relative_to(&variants_path, index_path)
.or_else(|_| std::path::absolute(&variants_path))
.map_err(LockErrorKind::DistributionRelativePath)?
.into_boxed_path();
VariantsJsonSource::Path { path }
} else {
let url = normalize_file_location(&variants_json.file.url)
.map_err(LockErrorKind::InvalidUrl)
.map_err(LockError::from)?;
VariantsJsonSource::Url { url }
}
}
};
Ok(Some(Self {
url,
hash: variants_json
.file
.hashes
.iter()
.max()
.cloned()
.map(Hash::from),
size: variants_json.file.size,
upload_time: variants_json
.file
.upload_time_utc_ms
.map(Timestamp::from_millisecond)
.transpose()
.map_err(LockErrorKind::InvalidTimestamp)?,
}))
} else {
Ok(None)
}
}
}
}
/// Returns the TOML representation of this variants JSON file.
fn to_toml(&self) -> Result<InlineTable, toml_edit::ser::Error> {
let mut table = InlineTable::new();
match &self.url {
VariantsJsonSource::Url { url } => {
table.insert("url", Value::from(url.as_ref()));
}
VariantsJsonSource::Path { path } => {
table.insert("path", Value::from(PortablePath::from(path).to_string()));
}
}
if let Some(hash) = &self.hash {
table.insert("hash", Value::from(hash.to_string()));
}
if let Some(size) = self.size {
table.insert(
"size",
toml_edit::ser::ValueSerializer::new().serialize_u64(size)?,
);
}
if let Some(upload_time) = self.upload_time {
table.insert("upload-time", Value::from(upload_time.to_string()));
}
Ok(table)
}
}
impl From<VariantsJsonWire> for VariantsJsonEntry {
fn from(wire: VariantsJsonWire) -> Self {
// TODO(konsti): Do we still need the wire type?
Self {
url: wire.url,
hash: wire.hash,
size: wire.size,
upload_time: wire.upload_time,
}
}
}
/// Inspired by: <https://discuss.python.org/t/lock-files-again-but-this-time-w-sdists/46593> /// Inspired by: <https://discuss.python.org/t/lock-files-again-but-this-time-w-sdists/46593>
#[derive(Clone, Debug, serde::Deserialize, PartialEq, Eq)] #[derive(Clone, Debug, serde::Deserialize, PartialEq, Eq)]
#[serde(try_from = "WheelWire")] #[serde(try_from = "WheelWire")]
@ -4739,6 +4979,33 @@ enum WheelWireSource {
}, },
} }
#[derive(Clone, Debug, serde::Deserialize, PartialEq, Eq)]
#[serde(untagged, rename_all = "kebab-case")]
enum VariantsJsonSource {
/// Used for all variants JSON files that come from remote sources.
Url {
/// A URL where the variants JSON file that was locked against was found. The location
/// does not need to exist in the future, so this should be treated as
/// only a hint to where to look and/or recording where the variants JSON file
/// originally came from.
url: UrlString,
},
/// Used for variants JSON files that come from local registries (like `--find-links`).
Path {
/// The path to the variants JSON file, relative to the index.
path: Box<Path>,
},
}
impl VariantsJsonSource {
fn url(&self) -> Option<&UrlString> {
match &self {
Self::Path { .. } => None,
Self::Url { url, .. } => Some(url),
}
}
}
impl Wheel { impl Wheel {
/// Returns the TOML representation of this wheel. /// Returns the TOML representation of this wheel.
fn to_toml(&self) -> Result<InlineTable, toml_edit::ser::Error> { fn to_toml(&self) -> Result<InlineTable, toml_edit::ser::Error> {
@ -5989,6 +6256,12 @@ enum LockErrorKind {
/// The ID of the workspace member with an invalid source. /// The ID of the workspace member with an invalid source.
id: PackageId, id: PackageId,
}, },
#[error("Failed to fetch and query variants for `{package_id}`")]
VariantError {
package_id: PackageId,
#[source]
err: uv_distribution::Error,
},
} }
/// An error that occurs when a source string could not be parsed. /// An error that occurs when a source string could not be parsed.

View File

@ -1,6 +1,7 @@
--- ---
source: crates/uv-resolver/src/lock/mod.rs source: crates/uv-resolver/src/lock/mod.rs
expression: result expression: result
snapshot_kind: text
--- ---
Ok( Ok(
Lock { Lock {
@ -90,6 +91,7 @@ Ok(
zstd: None, zstd: None,
}, },
], ],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [], dependencies: [],
optional_dependencies: {}, optional_dependencies: {},

View File

@ -1,6 +1,7 @@
--- ---
source: crates/uv-resolver/src/lock/mod.rs source: crates/uv-resolver/src/lock/mod.rs
expression: result expression: result
snapshot_kind: text
--- ---
Ok( Ok(
Lock { Lock {
@ -97,6 +98,7 @@ Ok(
zstd: None, zstd: None,
}, },
], ],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [], dependencies: [],
optional_dependencies: {}, optional_dependencies: {},

View File

@ -1,6 +1,7 @@
--- ---
source: crates/uv-resolver/src/lock/mod.rs source: crates/uv-resolver/src/lock/mod.rs
expression: result expression: result
snapshot_kind: text
--- ---
Ok( Ok(
Lock { Lock {
@ -93,6 +94,7 @@ Ok(
zstd: None, zstd: None,
}, },
], ],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [], dependencies: [],
optional_dependencies: {}, optional_dependencies: {},

View File

@ -1,6 +1,7 @@
--- ---
source: crates/uv-resolver/src/lock/mod.rs source: crates/uv-resolver/src/lock/mod.rs
expression: result expression: result
snapshot_kind: text
--- ---
Ok( Ok(
Lock { Lock {
@ -82,6 +83,7 @@ Ok(
}, },
), ),
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [], dependencies: [],
optional_dependencies: {}, optional_dependencies: {},
@ -130,6 +132,7 @@ Ok(
}, },
), ),
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [ dependencies: [
Dependency { Dependency {

View File

@ -1,6 +1,7 @@
--- ---
source: crates/uv-resolver/src/lock/mod.rs source: crates/uv-resolver/src/lock/mod.rs
expression: result expression: result
snapshot_kind: text
--- ---
Ok( Ok(
Lock { Lock {
@ -82,6 +83,7 @@ Ok(
}, },
), ),
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [], dependencies: [],
optional_dependencies: {}, optional_dependencies: {},
@ -130,6 +132,7 @@ Ok(
}, },
), ),
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [ dependencies: [
Dependency { Dependency {

View File

@ -1,6 +1,7 @@
--- ---
source: crates/uv-resolver/src/lock/mod.rs source: crates/uv-resolver/src/lock/mod.rs
expression: result expression: result
snapshot_kind: text
--- ---
Ok( Ok(
Lock { Lock {
@ -56,6 +57,7 @@ Ok(
}, },
sdist: None, sdist: None,
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [], dependencies: [],
optional_dependencies: {}, optional_dependencies: {},
@ -104,6 +106,7 @@ Ok(
}, },
), ),
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [], dependencies: [],
optional_dependencies: {}, optional_dependencies: {},
@ -152,6 +155,7 @@ Ok(
}, },
), ),
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [ dependencies: [
Dependency { Dependency {

View File

@ -1,6 +1,7 @@
--- ---
source: crates/uv-resolver/src/lock/mod.rs source: crates/uv-resolver/src/lock/mod.rs
expression: result expression: result
snapshot_kind: text
--- ---
Ok( Ok(
Lock { Lock {
@ -82,6 +83,7 @@ Ok(
}, },
), ),
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [], dependencies: [],
optional_dependencies: {}, optional_dependencies: {},
@ -130,6 +132,7 @@ Ok(
}, },
), ),
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [ dependencies: [
Dependency { Dependency {

View File

@ -1,6 +1,7 @@
--- ---
source: crates/uv-resolver/src/lock/mod.rs source: crates/uv-resolver/src/lock/mod.rs
expression: result expression: result
snapshot_kind: text
--- ---
Ok( Ok(
Lock { Lock {
@ -65,6 +66,7 @@ Ok(
}, },
sdist: None, sdist: None,
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [], dependencies: [],
optional_dependencies: {}, optional_dependencies: {},

View File

@ -1,6 +1,7 @@
--- ---
source: crates/uv-resolver/src/lock/mod.rs source: crates/uv-resolver/src/lock/mod.rs
expression: result expression: result
snapshot_kind: text
--- ---
Ok( Ok(
Lock { Lock {
@ -63,6 +64,7 @@ Ok(
}, },
sdist: None, sdist: None,
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [], dependencies: [],
optional_dependencies: {}, optional_dependencies: {},

View File

@ -1,6 +1,7 @@
--- ---
source: crates/uv-resolver/src/lock/mod.rs source: crates/uv-resolver/src/lock/mod.rs
expression: result expression: result
snapshot_kind: text
--- ---
Ok( Ok(
Lock { Lock {
@ -58,6 +59,7 @@ Ok(
}, },
sdist: None, sdist: None,
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [], dependencies: [],
optional_dependencies: {}, optional_dependencies: {},

View File

@ -1,6 +1,7 @@
--- ---
source: crates/uv-resolver/src/lock/mod.rs source: crates/uv-resolver/src/lock/mod.rs
expression: result expression: result
snapshot_kind: text
--- ---
Ok( Ok(
Lock { Lock {
@ -58,6 +59,7 @@ Ok(
}, },
sdist: None, sdist: None,
wheels: [], wheels: [],
variants_json: None,
fork_markers: [], fork_markers: [],
dependencies: [], dependencies: [],
optional_dependencies: {}, optional_dependencies: {},

View File

@ -13,7 +13,7 @@ use uv_configuration::DependencyGroupsWithDefaults;
use uv_console::human_readable_bytes; use uv_console::human_readable_bytes;
use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_normalize::{ExtraName, GroupName, PackageName};
use uv_pep440::Version; use uv_pep440::Version;
use uv_pep508::MarkerTree; use uv_pep508::{MarkerTree, MarkerVariantsUniversal};
use uv_pypi_types::ResolverMarkerEnvironment; use uv_pypi_types::ResolverMarkerEnvironment;
use crate::lock::PackageId; use crate::lock::PackageId;
@ -196,7 +196,9 @@ impl<'env> TreeDisplay<'env> {
if marker.is_false() { if marker.is_false() {
continue; continue;
} }
if markers.is_some_and(|markers| !marker.evaluate(markers, &[])) { if markers.is_some_and(|markers| {
!marker.evaluate(markers, &MarkerVariantsUniversal, &[])
}) {
continue; continue;
} }
// Add the package to the graph. // Add the package to the graph.
@ -233,7 +235,9 @@ impl<'env> TreeDisplay<'env> {
if marker.is_false() { if marker.is_false() {
continue; continue;
} }
if markers.is_some_and(|markers| !marker.evaluate(markers, &[])) { if markers.is_some_and(|markers| {
!marker.evaluate(markers, &MarkerVariantsUniversal, &[])
}) {
continue; continue;
} }
// Add the package to the graph. // Add the package to the graph.

View File

@ -6,6 +6,7 @@ use either::Either;
use uv_configuration::{Constraints, Excludes, Overrides}; use uv_configuration::{Constraints, Excludes, Overrides};
use uv_distribution_types::Requirement; use uv_distribution_types::Requirement;
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_pep508::MarkerVariantsUniversal;
use uv_types::RequestedRequirements; use uv_types::RequestedRequirements;
use crate::preferences::Preferences; use crate::preferences::Preferences;
@ -130,8 +131,11 @@ impl Manifest {
.apply(lookahead.requirements()) .apply(lookahead.requirements())
.filter(|requirement| !self.excludes.contains(&requirement.name)) .filter(|requirement| !self.excludes.contains(&requirement.name))
.filter(move |requirement| { .filter(move |requirement| {
requirement requirement.evaluate_markers(
.evaluate_markers(env.marker_environment(), lookahead.extras()) env.marker_environment(),
&MarkerVariantsUniversal,
lookahead.extras(),
)
}) })
}) })
.chain( .chain(
@ -139,7 +143,11 @@ impl Manifest {
.apply(&self.requirements) .apply(&self.requirements)
.filter(|requirement| !self.excludes.contains(&requirement.name)) .filter(|requirement| !self.excludes.contains(&requirement.name))
.filter(move |requirement| { .filter(move |requirement| {
requirement.evaluate_markers(env.marker_environment(), &[]) requirement.evaluate_markers(
env.marker_environment(),
&MarkerVariantsUniversal,
&[],
)
}), }),
) )
.chain( .chain(
@ -147,7 +155,11 @@ impl Manifest {
.requirements() .requirements()
.filter(|requirement| !self.excludes.contains(&requirement.name)) .filter(|requirement| !self.excludes.contains(&requirement.name))
.filter(move |requirement| { .filter(move |requirement| {
requirement.evaluate_markers(env.marker_environment(), &[]) requirement.evaluate_markers(
env.marker_environment(),
&MarkerVariantsUniversal,
&[],
)
}) })
.map(Cow::Borrowed), .map(Cow::Borrowed),
), ),
@ -159,7 +171,11 @@ impl Manifest {
.chain(self.constraints.requirements().map(Cow::Borrowed)) .chain(self.constraints.requirements().map(Cow::Borrowed))
.filter(|requirement| !self.excludes.contains(&requirement.name)) .filter(|requirement| !self.excludes.contains(&requirement.name))
.filter(move |requirement| { .filter(move |requirement| {
requirement.evaluate_markers(env.marker_environment(), &[]) requirement.evaluate_markers(
env.marker_environment(),
&MarkerVariantsUniversal,
&[],
)
}), }),
), ),
} }
@ -178,7 +194,11 @@ impl Manifest {
.requirements() .requirements()
.filter(|requirement| !self.excludes.contains(&requirement.name)) .filter(|requirement| !self.excludes.contains(&requirement.name))
.filter(move |requirement| { .filter(move |requirement| {
requirement.evaluate_markers(env.marker_environment(), &[]) requirement.evaluate_markers(
env.marker_environment(),
&MarkerVariantsUniversal,
&[],
)
}) })
.map(Cow::Borrowed), .map(Cow::Borrowed),
), ),
@ -188,7 +208,11 @@ impl Manifest {
.requirements() .requirements()
.filter(|requirement| !self.excludes.contains(&requirement.name)) .filter(|requirement| !self.excludes.contains(&requirement.name))
.filter(move |requirement| { .filter(move |requirement| {
requirement.evaluate_markers(env.marker_environment(), &[]) requirement.evaluate_markers(
env.marker_environment(),
&MarkerVariantsUniversal,
&[],
)
}) })
.map(Cow::Borrowed), .map(Cow::Borrowed),
), ),
@ -213,31 +237,44 @@ impl Manifest {
match mode { match mode {
// Include direct requirements, dependencies of editables, and transitive dependencies // Include direct requirements, dependencies of editables, and transitive dependencies
// of local packages. // of local packages.
DependencyMode::Transitive => Either::Left( DependencyMode::Transitive => {
self.lookaheads Either::Left(
.iter() self.lookaheads
.filter(|lookahead| lookahead.direct()) .iter()
.flat_map(move |lookahead| { .filter(|lookahead| lookahead.direct())
self.overrides .flat_map(move |lookahead| {
.apply(lookahead.requirements()) self.overrides.apply(lookahead.requirements()).filter(
.filter(move |requirement| { move |requirement| {
requirement requirement.evaluate_markers(
.evaluate_markers(env.marker_environment(), lookahead.extras()) env.marker_environment(),
}) &MarkerVariantsUniversal,
}) lookahead.extras(),
.chain( )
self.overrides },
.apply(&self.requirements) )
.filter(move |requirement| { })
requirement.evaluate_markers(env.marker_environment(), &[]) .chain(self.overrides.apply(&self.requirements).filter(
}), move |requirement| {
), requirement.evaluate_markers(
), env.marker_environment(),
&MarkerVariantsUniversal,
&[],
)
},
)),
)
}
// Restrict to the direct requirements. // Restrict to the direct requirements.
DependencyMode::Direct => { DependencyMode::Direct => {
Either::Right(self.overrides.apply(self.requirements.iter()).filter( Either::Right(self.overrides.apply(self.requirements.iter()).filter(
move |requirement| requirement.evaluate_markers(env.marker_environment(), &[]), move |requirement| {
requirement.evaluate_markers(
env.marker_environment(),
&MarkerVariantsUniversal,
&[],
)
},
)) ))
} }
} }
@ -254,7 +291,13 @@ impl Manifest {
) -> impl Iterator<Item = Cow<'a, Requirement>> + 'a { ) -> impl Iterator<Item = Cow<'a, Requirement>> + 'a {
self.overrides self.overrides
.apply(self.requirements.iter()) .apply(self.requirements.iter())
.filter(move |requirement| requirement.evaluate_markers(env.marker_environment(), &[])) .filter(move |requirement| {
requirement.evaluate_markers(
env.marker_environment(),
&MarkerVariantsUniversal,
&[],
)
})
} }
/// Apply the overrides and constraints to a set of requirements. /// Apply the overrides and constraints to a set of requirements.

View File

@ -49,12 +49,12 @@ pub(crate) fn requires_python(tree: MarkerTree) -> Option<RequiresPythonRange> {
collect_python_markers(tree, markers, range); collect_python_markers(tree, markers, range);
} }
} }
MarkerTreeKind::Extra(marker) => { MarkerTreeKind::List(marker) => {
for (_, tree) in marker.children() { for (_, tree) in marker.children() {
collect_python_markers(tree, markers, range); collect_python_markers(tree, markers, range);
} }
} }
MarkerTreeKind::List(marker) => { MarkerTreeKind::Extra(marker) => {
for (_, tree) in marker.children() { for (_, tree) in marker.children() {
collect_python_markers(tree, markers, range); collect_python_markers(tree, markers, range);
} }

View File

@ -7,7 +7,7 @@ use tracing::trace;
use uv_distribution_types::{IndexUrl, InstalledDist, InstalledDistKind}; use uv_distribution_types::{IndexUrl, InstalledDist, InstalledDistKind};
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_pep440::{Operator, Version}; use uv_pep440::{Operator, Version};
use uv_pep508::{MarkerTree, VerbatimUrl, VersionOrUrl}; use uv_pep508::{MarkerTree, MarkerVariantsUniversal, VerbatimUrl, VersionOrUrl};
use uv_pypi_types::{HashDigest, HashDigests, HashError}; use uv_pypi_types::{HashDigest, HashDigests, HashError};
use uv_requirements_txt::{RequirementEntry, RequirementsTxtRequirement}; use uv_requirements_txt::{RequirementEntry, RequirementsTxtRequirement};
@ -241,7 +241,10 @@ impl Preferences {
for preference in preferences { for preference in preferences {
// Filter non-matching preferences when resolving for an environment. // Filter non-matching preferences when resolving for an environment.
if let Some(markers) = env.marker_environment() { if let Some(markers) = env.marker_environment() {
if !preference.marker.evaluate(markers, &[]) { if !preference
.marker
.evaluate(markers, &MarkerVariantsUniversal, &[])
{
trace!("Excluding {preference} from preferences due to unmatched markers"); trace!("Excluding {preference} from preferences due to unmatched markers");
continue; continue;
} }

View File

@ -7,7 +7,7 @@ use rustc_hash::{FxBuildHasher, FxHashMap};
use uv_distribution_types::{DistributionMetadata, Name, SourceAnnotation, SourceAnnotations}; use uv_distribution_types::{DistributionMetadata, Name, SourceAnnotation, SourceAnnotations};
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_pep508::MarkerTree; use uv_pep508::{MarkerTree, MarkerVariantsUniversal};
use crate::resolution::{RequirementsTxtDist, ResolutionGraphNode}; use crate::resolution::{RequirementsTxtDist, ResolutionGraphNode};
use crate::{ResolverEnvironment, ResolverOutput}; use crate::{ResolverEnvironment, ResolverOutput};
@ -91,7 +91,11 @@ impl std::fmt::Display for DisplayResolutionGraph<'_> {
let mut sources = SourceAnnotations::default(); let mut sources = SourceAnnotations::default();
for requirement in self.resolution.requirements.iter().filter(|requirement| { for requirement in self.resolution.requirements.iter().filter(|requirement| {
requirement.evaluate_markers(self.env.marker_environment(), &[]) requirement.evaluate_markers(
self.env.marker_environment(),
&MarkerVariantsUniversal,
&[],
)
}) { }) {
if let Some(origin) = &requirement.origin { if let Some(origin) = &requirement.origin {
sources.add( sources.add(
@ -106,7 +110,11 @@ impl std::fmt::Display for DisplayResolutionGraph<'_> {
.constraints .constraints
.requirements() .requirements()
.filter(|requirement| { .filter(|requirement| {
requirement.evaluate_markers(self.env.marker_environment(), &[]) requirement.evaluate_markers(
self.env.marker_environment(),
&MarkerVariantsUniversal,
&[],
)
}) })
{ {
if let Some(origin) = &requirement.origin { if let Some(origin) = &requirement.origin {
@ -122,7 +130,11 @@ impl std::fmt::Display for DisplayResolutionGraph<'_> {
.overrides .overrides
.requirements() .requirements()
.filter(|requirement| { .filter(|requirement| {
requirement.evaluate_markers(self.env.marker_environment(), &[]) requirement.evaluate_markers(
self.env.marker_environment(),
&MarkerVariantsUniversal,
&[],
)
}) })
{ {
if let Some(origin) = &requirement.origin { if let Some(origin) = &requirement.origin {

View File

@ -449,6 +449,8 @@ impl ResolverOutput {
( (
ResolvedDist::Installable { ResolvedDist::Installable {
dist: Arc::new(dist), dist: Arc::new(dist),
// Only registry distributions have a variants JSON file.
variants_json: None,
version: Some(version.clone()), version: Some(version.clone()),
}, },
hashes, hashes,
@ -645,7 +647,7 @@ impl ResolverOutput {
) -> Result<MarkerTree, Box<ParsedUrlError>> { ) -> Result<MarkerTree, Box<ParsedUrlError>> {
use uv_pep508::{ use uv_pep508::{
CanonicalMarkerValueString, CanonicalMarkerValueVersion, MarkerExpression, CanonicalMarkerValueString, CanonicalMarkerValueVersion, MarkerExpression,
MarkerOperator, MarkerTree, MarkerOperator, MarkerTree, MarkerValueList,
}; };
/// A subset of the possible marker values. /// A subset of the possible marker values.
@ -657,6 +659,7 @@ impl ResolverOutput {
enum MarkerParam { enum MarkerParam {
Version(CanonicalMarkerValueVersion), Version(CanonicalMarkerValueVersion),
String(CanonicalMarkerValueString), String(CanonicalMarkerValueString),
List(MarkerValueList),
} }
/// Add all marker parameters from the given tree to the given set. /// Add all marker parameters from the given tree to the given set.
@ -688,6 +691,13 @@ impl ResolverOutput {
add_marker_params_from_tree(tree, set); add_marker_params_from_tree(tree, set);
} }
} }
MarkerTreeKind::List(marker) => {
// TODO(konsti): Do we care about this set here?
set.insert(MarkerParam::List(marker.key()));
for (_, tree) in marker.children() {
add_marker_params_from_tree(tree, set);
}
}
// We specifically don't care about these for the // We specifically don't care about these for the
// purposes of generating a marker string for a lock // purposes of generating a marker string for a lock
// file. Quoted strings are marker values given by the // file. Quoted strings are marker values given by the
@ -698,11 +708,6 @@ impl ResolverOutput {
add_marker_params_from_tree(tree, set); add_marker_params_from_tree(tree, set);
} }
} }
MarkerTreeKind::List(marker) => {
for (_, tree) in marker.children() {
add_marker_params_from_tree(tree, set);
}
}
} }
} }
@ -756,13 +761,29 @@ impl ResolverOutput {
} }
} }
MarkerParam::String(value_string) => { MarkerParam::String(value_string) => {
let from_env = marker_env.get_string(value_string); // TODO(konsti): What's the correct handling for `variant_label`?
let from_env = marker_env.get_string(value_string).unwrap_or("");
MarkerExpression::String { MarkerExpression::String {
key: value_string.into(), key: value_string.into(),
operator: MarkerOperator::Equal, operator: MarkerOperator::Equal,
value: from_env.into(), value: from_env.into(),
} }
} }
MarkerParam::List(value_variant) => {
match value_variant {
MarkerValueList::VariantNamespaces
| MarkerValueList::VariantFeatures
| MarkerValueList::VariantProperties => {
// We ignore variants for the resolution marker tree since they are package
// specific.
continue;
}
MarkerValueList::Extras | MarkerValueList::DependencyGroups => {
// TODO(konsti): Do we need to track them?
continue;
}
}
}
}; };
conjunction.and(MarkerTree::expression(expr)); conjunction.and(MarkerTree::expression(expr));
} }

View File

@ -505,8 +505,16 @@ pub(crate) enum ForkingPossibility<'d> {
} }
impl<'d> ForkingPossibility<'d> { impl<'d> ForkingPossibility<'d> {
pub(crate) fn new(env: &ResolverEnvironment, dep: &'d PubGrubDependency) -> Self { pub(crate) fn new(
let marker = dep.package.marker(); env: &ResolverEnvironment,
dep: &'d PubGrubDependency,
variant_base: Option<&str>,
) -> Self {
let marker = if let Some(variant_base) = variant_base {
dep.package.marker().with_variant_base(variant_base)
} else {
dep.package.marker()
};
if !env.included_by_marker(marker) { if !env.included_by_marker(marker) {
ForkingPossibility::DependencyAlwaysExcluded ForkingPossibility::DependencyAlwaysExcluded
} else if marker.is_true() { } else if marker.is_true() {
@ -576,8 +584,12 @@ impl Forker<'_> {
/// Returns true if the dependency represented by this forker may be /// Returns true if the dependency represented by this forker may be
/// included in the given resolver environment. /// included in the given resolver environment.
pub(crate) fn included(&self, env: &ResolverEnvironment) -> bool { pub(crate) fn included(&self, env: &ResolverEnvironment, variant_base: Option<&str>) -> bool {
let marker = self.package.marker(); let marker = if let Some(variant_base) = variant_base {
self.package.marker().with_variant_base(variant_base)
} else {
self.package.marker()
};
env.included_by_marker(marker) env.included_by_marker(marker)
} }
} }

View File

@ -2,9 +2,12 @@ use std::hash::BuildHasherDefault;
use std::sync::Arc; use std::sync::Arc;
use rustc_hash::FxHasher; use rustc_hash::FxHasher;
use uv_distribution::PackageVariantCache;
use uv_distribution_types::{IndexUrl, VersionId}; use uv_distribution_types::{IndexUrl, VersionId};
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_once_map::OnceMap; use uv_once_map::OnceMap;
use uv_variants::cache::VariantProviderCache;
use crate::resolver::provider::{MetadataResponse, VersionsResponse}; use crate::resolver::provider::{MetadataResponse, VersionsResponse};
@ -22,6 +25,12 @@ struct SharedInMemoryIndex {
/// A map from package ID to metadata for that distribution. /// A map from package ID to metadata for that distribution.
distributions: FxOnceMap<VersionId, Arc<MetadataResponse>>, distributions: FxOnceMap<VersionId, Arc<MetadataResponse>>,
/// The resolved variants, indexed by provider.
variant_providers: VariantProviderCache,
/// The resolved variant priorities, indexed by package version.
variant_priorities: PackageVariantCache,
} }
pub(crate) type FxOnceMap<K, V> = OnceMap<K, V, BuildHasherDefault<FxHasher>>; pub(crate) type FxOnceMap<K, V> = OnceMap<K, V, BuildHasherDefault<FxHasher>>;
@ -41,4 +50,14 @@ impl InMemoryIndex {
pub fn distributions(&self) -> &FxOnceMap<VersionId, Arc<MetadataResponse>> { pub fn distributions(&self) -> &FxOnceMap<VersionId, Arc<MetadataResponse>> {
&self.0.distributions &self.0.distributions
} }
/// Returns a reference to the variant providers map.
pub fn variant_providers(&self) -> &VariantProviderCache {
&self.0.variant_providers
}
/// Returns a reference to the variant priorities map.
pub fn variant_priorities(&self) -> &PackageVariantCache {
&self.0.variant_priorities
}
} }

View File

@ -24,15 +24,17 @@ use uv_configuration::{Constraints, Excludes, Overrides};
use uv_distribution::{ArchiveMetadata, DistributionDatabase}; use uv_distribution::{ArchiveMetadata, DistributionDatabase};
use uv_distribution_types::{ use uv_distribution_types::{
BuiltDist, CompatibleDist, DerivationChain, Dist, DistErrorKind, DistributionMetadata, BuiltDist, CompatibleDist, DerivationChain, Dist, DistErrorKind, DistributionMetadata,
IncompatibleDist, IncompatibleSource, IncompatibleWheel, IndexCapabilities, IndexLocations, GlobalVersionId, IncompatibleDist, IncompatibleSource, IncompatibleWheel, IndexCapabilities,
IndexMetadata, IndexUrl, InstalledDist, Name, PythonRequirementKind, RemoteSource, Requirement, IndexLocations, IndexMetadata, IndexUrl, InstalledDist, Name, PackageId, PrioritizedDist,
ResolvedDist, ResolvedDistRef, SourceDist, VersionOrUrlRef, implied_markers, PythonRequirementKind, RegistryVariantsJson, RemoteSource, Requirement, ResolvedDist,
ResolvedDistRef, SourceDist, VersionId, VersionOrUrlRef, implied_markers,
}; };
use uv_git::GitResolver; use uv_git::GitResolver;
use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_normalize::{ExtraName, GroupName, PackageName};
use uv_pep440::{MIN_VERSION, Version, VersionSpecifiers, release_specifiers_to_ranges}; use uv_pep440::{MIN_VERSION, Version, VersionSpecifiers, release_specifiers_to_ranges};
use uv_pep508::{ use uv_pep508::{
MarkerEnvironment, MarkerExpression, MarkerOperator, MarkerTree, MarkerValueString, MarkerEnvironment, MarkerExpression, MarkerOperator, MarkerTree, MarkerValueString,
MarkerVariantsEnvironment, MarkerVariantsUniversal,
}; };
use uv_platform_tags::{IncompatibleTag, Tags}; use uv_platform_tags::{IncompatibleTag, Tags};
use uv_pypi_types::{ConflictItem, ConflictItemRef, ConflictKindRef, Conflicts, VerbatimParsedUrl}; use uv_pypi_types::{ConflictItem, ConflictItemRef, ConflictKindRef, Conflicts, VerbatimParsedUrl};
@ -71,7 +73,7 @@ pub use crate::resolver::index::InMemoryIndex;
use crate::resolver::indexes::Indexes; use crate::resolver::indexes::Indexes;
pub use crate::resolver::provider::{ pub use crate::resolver::provider::{
DefaultResolverProvider, MetadataResponse, PackageVersionsResult, ResolverProvider, DefaultResolverProvider, MetadataResponse, PackageVersionsResult, ResolverProvider,
VersionsResponse, WheelMetadataResult, VariantProviderResult, VersionsResponse, WheelMetadataResult,
}; };
pub use crate::resolver::reporter::{BuildId, Reporter}; pub use crate::resolver::reporter::{BuildId, Reporter};
use crate::resolver::system::SystemDependency; use crate::resolver::system::SystemDependency;
@ -83,6 +85,8 @@ use crate::{
marker, marker,
}; };
pub(crate) use provider::MetadataUnavailable; pub(crate) use provider::MetadataUnavailable;
use uv_variants::resolved_variants::ResolvedVariants;
use uv_variants::variant_with_label::VariantWithLabel;
mod availability; mod availability;
mod batch_prefetch; mod batch_prefetch;
@ -617,6 +621,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
&state.pins, &state.pins,
&state.fork_urls, &state.fork_urls,
&state.env, &state.env,
&self.index,
&state.python_requirement, &state.python_requirement,
&state.pubgrub, &state.pubgrub,
)?; )?;
@ -1195,11 +1200,13 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
if env.marker_environment().is_none() && !self.options.required_environments.is_empty() if env.marker_environment().is_none() && !self.options.required_environments.is_empty()
{ {
let wheel_marker = implied_markers(filename); let wheel_marker = implied_markers(filename);
let variant_base = PackageId::from_url(&url.verbatim.to_url()).to_string();
// If the user explicitly marked a platform as required, ensure it has coverage. // If the user explicitly marked a platform as required, ensure it has coverage.
for environment_marker in self.options.required_environments.iter().copied() { for environment_marker in self.options.required_environments.iter().copied() {
// If the platform is part of the current environment... // If the platform is part of the current environment...
if env.included_by_marker(environment_marker) if env.included_by_marker(environment_marker)
&& !find_environments(id, pubgrub).is_disjoint(environment_marker) && !find_environments(id, pubgrub, &variant_base)
.is_disjoint(environment_marker)
{ {
// ...but the wheel doesn't support it, it's incompatible. // ...but the wheel doesn't support it, it's incompatible.
if wheel_marker.is_disjoint(environment_marker) { if wheel_marker.is_disjoint(environment_marker) {
@ -1328,6 +1335,15 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
return Ok(None); return Ok(None);
}; };
// TODO(konsti): Can we make this an option so we don't pay any allocations?
let mut variant_prioritized_dist_binding = PrioritizedDist::default();
let candidate = self.variant_candidate(
candidate,
env,
request_sink,
&mut variant_prioritized_dist_binding,
)?;
let dist = match candidate.dist() { let dist = match candidate.dist() {
CandidateDist::Compatible(dist) => dist, CandidateDist::Compatible(dist) => dist,
CandidateDist::Incompatible { CandidateDist::Incompatible {
@ -1429,6 +1445,63 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
Ok(Some(ResolverVersion::Unforked(version))) Ok(Some(ResolverVersion::Unforked(version)))
} }
fn variant_candidate<'prioritized>(
&self,
candidate: Candidate<'prioritized>,
env: &ResolverEnvironment,
request_sink: &Sender<Request>,
variant_prioritized_dist_binding: &'prioritized mut PrioritizedDist,
) -> Result<Candidate<'prioritized>, ResolveError> {
let candidate = if env.marker_environment().is_some() {
// When solving for a specific environment, check if there is a matching variant wheel
// for the current environment.
// TODO(konsti): When solving for an environment that is not the current host, don't
// consider variants unless a static variant is given.
let Some(prioritized_dist) = candidate.prioritized() else {
return Ok(candidate);
};
// No `variants.json`, no variants.
// TODO(konsti): Be more lenient, e.g. parse the wheel itself?
let Some(variants_json) = prioritized_dist.variants_json() else {
return Ok(candidate);
};
// If the distribution is not indexed, we can't resolve variants.
let Some(index) = prioritized_dist.index() else {
return Ok(candidate);
};
// Query the host for the applicable features and properties.
let version_id = GlobalVersionId::new(
VersionId::NameVersion(candidate.name().clone(), candidate.version().clone()),
index.clone(),
);
if self.index.variant_priorities().register(version_id.clone()) {
request_sink
.blocking_send(Request::Variants(version_id.clone(), variants_json.clone()))?;
}
let resolved_variants = self.index.variant_priorities().wait_blocking(&version_id);
let Some(resolved_variants) = &resolved_variants else {
panic!("We have variants, why didn't they resolve?");
};
let Some(variant_prioritized_dist) =
prioritized_dist.prioritize_best_variant_wheel(resolved_variants)
else {
return Ok(candidate);
};
*variant_prioritized_dist_binding = variant_prioritized_dist;
candidate.prioritize_best_variant_wheel(variant_prioritized_dist_binding)
} else {
// In universal mode, a variant wheel with an otherwise compatible tag is acceptable.
candidate.allow_variant_wheels()
};
Ok(candidate)
}
/// Determine whether a candidate covers all supported platforms; and, if not, generate a fork. /// Determine whether a candidate covers all supported platforms; and, if not, generate a fork.
/// ///
/// This only ever applies to versions that lack source distributions And, for now, we only /// This only ever applies to versions that lack source distributions And, for now, we only
@ -1468,13 +1541,15 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
return Ok(None); return Ok(None);
} }
let variant_base = candidate.package_id().to_string();
// If the user explicitly marked a platform as required, ensure it has coverage. // If the user explicitly marked a platform as required, ensure it has coverage.
for marker in self.options.required_environments.iter().copied() { for marker in self.options.required_environments.iter().copied() {
// If the platform is part of the current environment... // If the platform is part of the current environment...
if env.included_by_marker(marker) { if env.included_by_marker(marker) {
// But isn't supported by the distribution... // But isn't supported by the distribution...
if dist.implied_markers().is_disjoint(marker) if dist.implied_markers().is_disjoint(marker)
&& !find_environments(id, pubgrub).is_disjoint(marker) && !find_environments(id, pubgrub, &variant_base).is_disjoint(marker)
{ {
// Then we need to fork. // Then we need to fork.
let Some((left, right)) = fork_version_by_marker(env, marker) else { let Some((left, right)) = fork_version_by_marker(env, marker) else {
@ -1543,6 +1618,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
) else { ) else {
return Ok(None); return Ok(None);
}; };
let CandidateDist::Compatible(base_dist) = base_candidate.dist() else { let CandidateDist::Compatible(base_dist) = base_candidate.dist() else {
return Ok(None); return Ok(None);
}; };
@ -1748,6 +1824,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
pins: &FilePins, pins: &FilePins,
fork_urls: &ForkUrls, fork_urls: &ForkUrls,
env: &ResolverEnvironment, env: &ResolverEnvironment,
in_memory_index: &InMemoryIndex,
python_requirement: &PythonRequirement, python_requirement: &PythonRequirement,
pubgrub: &State<UvDependencyProvider>, pubgrub: &State<UvDependencyProvider>,
) -> Result<ForkedDependencies, ResolveError> { ) -> Result<ForkedDependencies, ResolveError> {
@ -1758,6 +1835,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
pins, pins,
fork_urls, fork_urls,
env, env,
in_memory_index,
python_requirement, python_requirement,
pubgrub, pubgrub,
); );
@ -1769,7 +1847,14 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
Dependencies::Unavailable(err) => ForkedDependencies::Unavailable(err), Dependencies::Unavailable(err) => ForkedDependencies::Unavailable(err),
}) })
} else { } else {
Ok(result?.fork(env, python_requirement, &self.conflicts)) // Grab the pinned distribution for the given name and version.
let variant_base = package.name().map(|name| format!("{name}=={version}"));
Ok(result?.fork(
env,
python_requirement,
&self.conflicts,
variant_base.as_deref(),
))
} }
} }
@ -1783,6 +1868,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
pins: &FilePins, pins: &FilePins,
fork_urls: &ForkUrls, fork_urls: &ForkUrls,
env: &ResolverEnvironment, env: &ResolverEnvironment,
in_memory_index: &InMemoryIndex,
python_requirement: &PythonRequirement, python_requirement: &PythonRequirement,
pubgrub: &State<UvDependencyProvider>, pubgrub: &State<UvDependencyProvider>,
) -> Result<Dependencies, ResolveError> { ) -> Result<Dependencies, ResolveError> {
@ -1797,6 +1883,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
None, None,
None, None,
env, env,
&MarkerVariantsUniversal,
python_requirement, python_requirement,
); );
@ -1830,6 +1917,10 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
}; };
let version_id = dist.version_id(); let version_id = dist.version_id();
// If we're resolving for a specific environment, use the host variants, otherwise resolve
// for all variants.
let variant = Self::variant_properties(name, version, pins, env, in_memory_index);
// If the package does not exist in the registry or locally, we cannot fetch its dependencies // If the package does not exist in the registry or locally, we cannot fetch its dependencies
if self.dependency_mode.is_transitive() if self.dependency_mode.is_transitive()
&& self.unavailable_packages.get(name).is_some() && self.unavailable_packages.get(name).is_some()
@ -1914,6 +2005,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
group.as_ref(), group.as_ref(),
Some(name), Some(name),
env, env,
&variant,
python_requirement, python_requirement,
); );
@ -2012,6 +2104,61 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
Ok(Dependencies::Available(dependencies)) Ok(Dependencies::Available(dependencies))
} }
fn variant_properties(
name: &PackageName,
version: &Version,
pins: &FilePins,
env: &ResolverEnvironment,
in_memory_index: &InMemoryIndex,
) -> VariantWithLabel {
// TODO(konsti): Perf/Caching with version selection: This is in the hot path!
if env.marker_environment().is_none() {
return VariantWithLabel::default();
}
// Grab the pinned distribution for the given name and version.
let Some(dist) = pins.get(name, version) else {
return VariantWithLabel::default();
};
let Some(filename) = dist.wheel_filename() else {
// TODO(konsti): Handle installed dists too
return VariantWithLabel::default();
};
let Some(variant_label) = filename.variant() else {
return VariantWithLabel::default();
};
let Some(index) = dist.index() else {
warn!("Wheel variant has no index: {filename}");
return VariantWithLabel::default();
};
let version_id = GlobalVersionId::new(
VersionId::NameVersion(name.clone(), version.clone()),
index.clone(),
);
let Some(resolved_variants) = in_memory_index.variant_priorities().get(&version_id) else {
return VariantWithLabel::default();
};
// Collect the host properties for marker filtering.
// TODO(konsti): We shouldn't need to clone
let variant = resolved_variants
.variants_json
.variants
.get(variant_label)
.expect("Missing previously select variant label");
VariantWithLabel {
variant: variant.clone(),
label: Some(variant_label.clone()),
}
}
/// The regular and dev dependencies filtered by Python version and the markers of this fork, /// The regular and dev dependencies filtered by Python version and the markers of this fork,
/// plus the extras dependencies of the current package (e.g., `black` depending on /// plus the extras dependencies of the current package (e.g., `black` depending on
/// `black[colorama]`). /// `black[colorama]`).
@ -2023,6 +2170,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
dev: Option<&'a GroupName>, dev: Option<&'a GroupName>,
name: Option<&PackageName>, name: Option<&PackageName>,
env: &'a ResolverEnvironment, env: &'a ResolverEnvironment,
variants: &'a impl MarkerVariantsEnvironment,
python_requirement: &'a PythonRequirement, python_requirement: &'a PythonRequirement,
) -> impl Iterator<Item = Cow<'a, Requirement>> { ) -> impl Iterator<Item = Cow<'a, Requirement>> {
let python_marker = python_requirement.to_marker_tree(); let python_marker = python_requirement.to_marker_tree();
@ -2034,6 +2182,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
dev_dependencies.get(dev).into_iter().flatten(), dev_dependencies.get(dev).into_iter().flatten(),
extra, extra,
env, env,
variants,
python_marker, python_marker,
python_requirement, python_requirement,
))) )))
@ -2046,6 +2195,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
dependencies.iter(), dependencies.iter(),
extra, extra,
env, env,
variants,
python_marker, python_marker,
python_requirement, python_requirement,
))) )))
@ -2055,6 +2205,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
dependencies.iter(), dependencies.iter(),
extra, extra,
env, env,
variants,
python_marker, python_marker,
python_requirement, python_requirement,
) )
@ -2076,6 +2227,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
dependencies, dependencies,
Some(&extra), Some(&extra),
env, env,
&variants,
python_marker, python_marker,
python_requirement, python_requirement,
) { ) {
@ -2145,6 +2297,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
dependencies: impl IntoIterator<Item = &'data Requirement> + 'parameters, dependencies: impl IntoIterator<Item = &'data Requirement> + 'parameters,
extra: Option<&'parameters ExtraName>, extra: Option<&'parameters ExtraName>,
env: &'parameters ResolverEnvironment, env: &'parameters ResolverEnvironment,
variants: &'parameters impl MarkerVariantsEnvironment,
python_marker: MarkerTree, python_marker: MarkerTree,
python_requirement: &'parameters PythonRequirement, python_requirement: &'parameters PythonRequirement,
) -> impl Iterator<Item = Cow<'data, Requirement>> + 'parameters ) -> impl Iterator<Item = Cow<'data, Requirement>> + 'parameters
@ -2158,6 +2311,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
requirement, requirement,
extra, extra,
env, env,
variants,
python_marker, python_marker,
python_requirement, python_requirement,
) )
@ -2167,18 +2321,20 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
requirement, requirement,
extra, extra,
env, env,
variants,
python_marker, python_marker,
python_requirement, python_requirement,
)) ))
}) })
} }
/// Whether a requirement is applicable for the Python version, the markers of this fork and the /// Whether a requirement is applicable for the Python version, the markers of this fork, the
/// requested extra. /// host variants if applicable and the requested extra.
fn is_requirement_applicable( fn is_requirement_applicable(
requirement: &Requirement, requirement: &Requirement,
extra: Option<&ExtraName>, extra: Option<&ExtraName>,
env: &ResolverEnvironment, env: &ResolverEnvironment,
variants: &impl MarkerVariantsEnvironment,
python_marker: MarkerTree, python_marker: MarkerTree,
python_requirement: &PythonRequirement, python_requirement: &PythonRequirement,
) -> bool { ) -> bool {
@ -2186,12 +2342,14 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
match extra { match extra {
Some(source_extra) => { Some(source_extra) => {
// Only include requirements that are relevant for the current extra. // Only include requirements that are relevant for the current extra.
if requirement.evaluate_markers(env.marker_environment(), &[]) { if requirement.evaluate_markers(env.marker_environment(), &variants, &[]) {
return false; return false;
} }
if !requirement if !requirement.evaluate_markers(
.evaluate_markers(env.marker_environment(), slice::from_ref(source_extra)) env.marker_environment(),
{ &variants,
slice::from_ref(source_extra),
) {
return false; return false;
} }
if !env.included_by_group(ConflictItemRef::from((&requirement.name, source_extra))) if !env.included_by_group(ConflictItemRef::from((&requirement.name, source_extra)))
@ -2200,7 +2358,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
} }
} }
None => { None => {
if !requirement.evaluate_markers(env.marker_environment(), &[]) { if !requirement.evaluate_markers(env.marker_environment(), variants, &[]) {
return false; return false;
} }
} }
@ -2233,6 +2391,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
requirement: Cow<'data, Requirement>, requirement: Cow<'data, Requirement>,
extra: Option<&'parameters ExtraName>, extra: Option<&'parameters ExtraName>,
env: &'parameters ResolverEnvironment, env: &'parameters ResolverEnvironment,
variants: &'parameters (impl MarkerVariantsEnvironment + 'parameters),
python_marker: MarkerTree, python_marker: MarkerTree,
python_requirement: &'parameters PythonRequirement, python_requirement: &'parameters PythonRequirement,
) -> impl Iterator<Item = Cow<'data, Requirement>> + 'parameters ) -> impl Iterator<Item = Cow<'data, Requirement>> + 'parameters
@ -2294,7 +2453,8 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
// constraint should only apply when _both_ markers are true. // constraint should only apply when _both_ markers are true.
if python_marker.is_disjoint(marker) { if python_marker.is_disjoint(marker) {
trace!( trace!(
"Skipping constraint {requirement} because of Requires-Python: {requires_python}" "Skipping constraint {requirement} \
because of Requires-Python: {requires_python}"
); );
return None; return None;
} }
@ -2323,18 +2483,22 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
// If the constraint isn't relevant for the current platform, skip it. // If the constraint isn't relevant for the current platform, skip it.
match extra { match extra {
Some(source_extra) => { Some(source_extra) => {
if !constraint if !constraint.evaluate_markers(
.evaluate_markers(env.marker_environment(), slice::from_ref(source_extra)) env.marker_environment(),
{ &variants,
slice::from_ref(source_extra),
) {
return None; return None;
} }
if !env.included_by_group(ConflictItemRef::from((&requirement.name, source_extra))) if !env.included_by_group(ConflictItemRef::from((
{ &requirement.name,
source_extra,
))) {
return None; return None;
} }
} }
None => { None => {
if !constraint.evaluate_markers(env.marker_environment(), &[]) { if !constraint.evaluate_markers(env.marker_environment(), &variants, &[]) {
return None; return None;
} }
} }
@ -2394,6 +2558,15 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
.distributions() .distributions()
.done(dist.version_id(), Arc::new(metadata)); .done(dist.version_id(), Arc::new(metadata));
} }
Some(Response::Variants {
version_id,
resolved_variants,
}) => {
trace!("Received variant metadata for: {version_id}");
self.index
.variant_priorities()
.done(version_id, Arc::new(resolved_variants));
}
None => {} None => {}
} }
} }
@ -2560,6 +2733,7 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
}; };
// If there is not a compatible distribution, short-circuit. // If there is not a compatible distribution, short-circuit.
// TODO(konsti): Consider prefetching variants instead.
let Some(dist) = candidate.compatible() else { let Some(dist) = candidate.compatible() else {
return Ok(None); return Ok(None);
}; };
@ -2680,9 +2854,32 @@ impl<InstalledPackages: InstalledPackagesProvider> ResolverState<InstalledPackag
Ok(None) Ok(None)
} }
} }
Request::Variants(version_id, variants_json) => self
.fetch_and_query_variants(variants_json, provider)
.await
.map(|resolved_variants| {
Some(Response::Variants {
version_id,
resolved_variants,
})
}),
} }
} }
async fn fetch_and_query_variants<Provider: ResolverProvider>(
&self,
variants_json: RegistryVariantsJson,
provider: &Provider,
) -> Result<ResolvedVariants, ResolveError> {
let Some(marker_env) = self.env.marker_environment() else {
unreachable!("Variants should only be queried in non-universal resolution")
};
provider
.fetch_and_query_variants(&variants_json, marker_env)
.await
.map_err(ResolveError::VariantFrontend)
}
fn convert_no_solution_err( fn convert_no_solution_err(
&self, &self,
mut err: pubgrub::NoSolutionError<UvDependencyProvider>, mut err: pubgrub::NoSolutionError<UvDependencyProvider>,
@ -3524,6 +3721,8 @@ pub(crate) enum Request {
Installed(InstalledDist), Installed(InstalledDist),
/// A request to pre-fetch the metadata for a package and the best-guess distribution. /// A request to pre-fetch the metadata for a package and the best-guess distribution.
Prefetch(PackageName, Range<Version>, PythonRequirement), Prefetch(PackageName, Range<Version>, PythonRequirement),
/// Resolve the variants for a package
Variants(GlobalVersionId, RegistryVariantsJson),
} }
impl<'a> From<ResolvedDistRef<'a>> for Request { impl<'a> From<ResolvedDistRef<'a>> for Request {
@ -3578,6 +3777,9 @@ impl Display for Request {
Self::Prefetch(package_name, range, _) => { Self::Prefetch(package_name, range, _) => {
write!(f, "Prefetch {package_name} {range}") write!(f, "Prefetch {package_name} {range}")
} }
Self::Variants(version_id, _) => {
write!(f, "Variants {version_id}")
}
} }
} }
} }
@ -3597,6 +3799,11 @@ enum Response {
dist: InstalledDist, dist: InstalledDist,
metadata: MetadataResponse, metadata: MetadataResponse,
}, },
/// The returned variant compatibility.
Variants {
version_id: GlobalVersionId,
resolved_variants: ResolvedVariants,
},
} }
/// Information about the dependencies for a particular package. /// Information about the dependencies for a particular package.
@ -3633,6 +3840,7 @@ impl Dependencies {
env: &ResolverEnvironment, env: &ResolverEnvironment,
python_requirement: &PythonRequirement, python_requirement: &PythonRequirement,
conflicts: &Conflicts, conflicts: &Conflicts,
variant_base: Option<&str>,
) -> ForkedDependencies { ) -> ForkedDependencies {
let deps = match self { let deps = match self {
Self::Available(deps) => deps, Self::Available(deps) => deps,
@ -3651,7 +3859,13 @@ impl Dependencies {
let Forks { let Forks {
mut forks, mut forks,
diverging_packages, diverging_packages,
} = Forks::new(name_to_deps, env, python_requirement, conflicts); } = Forks::new(
name_to_deps,
env,
python_requirement,
conflicts,
variant_base,
);
if forks.is_empty() { if forks.is_empty() {
ForkedDependencies::Unforked(vec![]) ForkedDependencies::Unforked(vec![])
} else if forks.len() == 1 { } else if forks.len() == 1 {
@ -3709,6 +3923,7 @@ impl Forks {
env: &ResolverEnvironment, env: &ResolverEnvironment,
python_requirement: &PythonRequirement, python_requirement: &PythonRequirement,
conflicts: &Conflicts, conflicts: &Conflicts,
variant_base: Option<&str>,
) -> Self { ) -> Self {
let python_marker = python_requirement.to_marker_tree(); let python_marker = python_requirement.to_marker_tree();
@ -3738,7 +3953,11 @@ impl Forks {
.is_none_or(|bound| !python_requirement.raises(&bound)) .is_none_or(|bound| !python_requirement.raises(&bound))
{ {
let dep = deps.pop().unwrap(); let dep = deps.pop().unwrap();
let marker = dep.package.marker(); let marker = if let Some(variant_base) = variant_base {
dep.package.marker().with_variant_base(variant_base)
} else {
dep.package.marker()
};
for fork in &mut forks { for fork in &mut forks {
if fork.env.included_by_marker(marker) { if fork.env.included_by_marker(marker) {
fork.add_dependency(dep.clone()); fork.add_dependency(dep.clone());
@ -3770,7 +3989,7 @@ impl Forks {
} }
} }
for dep in deps { for dep in deps {
let mut forker = match ForkingPossibility::new(env, &dep) { let mut forker = match ForkingPossibility::new(env, &dep, variant_base) {
ForkingPossibility::Possible(forker) => forker, ForkingPossibility::Possible(forker) => forker,
ForkingPossibility::DependencyAlwaysExcluded => { ForkingPossibility::DependencyAlwaysExcluded => {
// If the markers can never be satisfied by the parent // If the markers can never be satisfied by the parent
@ -3799,12 +4018,12 @@ impl Forks {
for fork_env in envs { for fork_env in envs {
let mut new_fork = fork.clone(); let mut new_fork = fork.clone();
new_fork.set_env(fork_env); new_fork.set_env(fork_env, variant_base);
// We only add the dependency to this fork if it // We only add the dependency to this fork if it
// satisfies the fork's markers. Some forks are // satisfies the fork's markers. Some forks are
// specifically created to exclude this dependency, // specifically created to exclude this dependency,
// so this isn't always true! // so this isn't always true!
if forker.included(&new_fork.env) { if forker.included(&new_fork.env, variant_base) {
new_fork.add_dependency(dep.clone()); new_fork.add_dependency(dep.clone());
} }
// Filter out any forks we created that are disjoint with our // Filter out any forks we created that are disjoint with our
@ -3943,10 +4162,14 @@ impl Fork {
/// ///
/// Any dependency in this fork that does not satisfy the given environment /// Any dependency in this fork that does not satisfy the given environment
/// is removed. /// is removed.
fn set_env(&mut self, env: ResolverEnvironment) { fn set_env(&mut self, env: ResolverEnvironment, variant_base: Option<&str>) {
self.env = env; self.env = env;
self.dependencies.retain(|dep| { self.dependencies.retain(|dep| {
let marker = dep.package.marker(); let marker = if let Some(variant_base) = variant_base {
dep.package.marker().with_variant_base(variant_base)
} else {
dep.package.marker()
};
if self.env.included_by_marker(marker) { if self.env.included_by_marker(marker) {
return true; return true;
} }
@ -4076,7 +4299,11 @@ fn enrich_dependency_error(
} }
/// Compute the set of markers for which a package is known to be relevant. /// Compute the set of markers for which a package is known to be relevant.
fn find_environments(id: Id<PubGrubPackage>, state: &State<UvDependencyProvider>) -> MarkerTree { fn find_environments(
id: Id<PubGrubPackage>,
state: &State<UvDependencyProvider>,
variant_base: &str,
) -> MarkerTree {
let package = &state.package_store[id]; let package = &state.package_store[id];
if package.is_root() { if package.is_root() {
return MarkerTree::TRUE; return MarkerTree::TRUE;
@ -4094,8 +4321,8 @@ fn find_environments(id: Id<PubGrubPackage>, state: &State<UvDependencyProvider>
if let Kind::FromDependencyOf(id1, _, id2, _) = &incompat.kind { if let Kind::FromDependencyOf(id1, _, id2, _) = &incompat.kind {
if id == *id2 { if id == *id2 {
marker.or({ marker.or({
let mut marker = package.marker(); let mut marker = package.marker().with_variant_base(variant_base);
marker.and(find_environments(*id1, state)); marker.and(find_environments(*id1, state, variant_base));
marker marker
}); });
} }

View File

@ -4,13 +4,14 @@ use uv_client::MetadataFormat;
use uv_configuration::BuildOptions; use uv_configuration::BuildOptions;
use uv_distribution::{ArchiveMetadata, DistributionDatabase, Reporter}; use uv_distribution::{ArchiveMetadata, DistributionDatabase, Reporter};
use uv_distribution_types::{ use uv_distribution_types::{
Dist, IndexCapabilities, IndexMetadata, IndexMetadataRef, InstalledDist, RequestedDist, Dist, IndexCapabilities, IndexMetadata, IndexMetadataRef, InstalledDist, RegistryVariantsJson,
RequiresPython, RequestedDist, RequiresPython,
}; };
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_pep440::{Version, VersionSpecifiers}; use uv_pep440::{Version, VersionSpecifiers};
use uv_platform_tags::Tags; use uv_platform_tags::Tags;
use uv_types::{BuildContext, HashStrategy}; use uv_types::{BuildContext, HashStrategy};
use uv_variants::resolved_variants::ResolvedVariants;
use crate::ExcludeNewer; use crate::ExcludeNewer;
use crate::flat_index::FlatIndex; use crate::flat_index::FlatIndex;
@ -19,6 +20,7 @@ use crate::yanks::AllowedYanks;
pub type PackageVersionsResult = Result<VersionsResponse, uv_client::Error>; pub type PackageVersionsResult = Result<VersionsResponse, uv_client::Error>;
pub type WheelMetadataResult = Result<MetadataResponse, uv_distribution::Error>; pub type WheelMetadataResult = Result<MetadataResponse, uv_distribution::Error>;
pub type VariantProviderResult = Result<ResolvedVariants, uv_distribution::Error>;
/// The response when requesting versions for a package /// The response when requesting versions for a package
#[derive(Debug)] #[derive(Debug)]
@ -100,6 +102,13 @@ pub trait ResolverProvider {
dist: &'io InstalledDist, dist: &'io InstalledDist,
) -> impl Future<Output = WheelMetadataResult> + 'io; ) -> impl Future<Output = WheelMetadataResult> + 'io;
/// Fetch the variants for a distribution given the marker environment.
fn fetch_and_query_variants<'io>(
&'io self,
variants_json: &'io RegistryVariantsJson,
marker_env: &'io uv_pep508::MarkerEnvironment,
) -> impl Future<Output = VariantProviderResult> + 'io;
/// Set the [`Reporter`] to use for this installer. /// Set the [`Reporter`] to use for this installer.
#[must_use] #[must_use]
fn with_reporter(self, reporter: Arc<dyn Reporter>) -> Self; fn with_reporter(self, reporter: Arc<dyn Reporter>) -> Self;
@ -308,6 +317,17 @@ impl<Context: BuildContext> ResolverProvider for DefaultResolverProvider<'_, Con
} }
} }
/// Fetch the variants for a distribution given the marker environment.
async fn fetch_and_query_variants<'io>(
&'io self,
variants_json: &'io RegistryVariantsJson,
marker_env: &'io uv_pep508::MarkerEnvironment,
) -> VariantProviderResult {
self.fetcher
.fetch_and_query_variants(variants_json, marker_env)
.await
}
/// Set the [`Reporter`] to use for this installer. /// Set the [`Reporter`] to use for this installer.
fn with_reporter(self, reporter: Arc<dyn Reporter>) -> Self { fn with_reporter(self, reporter: Arc<dyn Reporter>) -> Self {
Self { Self {

View File

@ -6,7 +6,10 @@ use itertools::Itertools;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_normalize::{ExtraName, GroupName, PackageName};
use uv_pep508::{ExtraOperator, MarkerEnvironment, MarkerExpression, MarkerOperator, MarkerTree}; use uv_pep508::{
ExtraOperator, MarkerEnvironment, MarkerExpression, MarkerOperator, MarkerTree,
MarkerVariantsEnvironment, MarkerVariantsUniversal,
};
use uv_pypi_types::{ConflictItem, ConflictKind, Conflicts, Inference}; use uv_pypi_types::{ConflictItem, ConflictKind, Conflicts, Inference};
use crate::ResolveError; use crate::ResolveError;
@ -293,7 +296,7 @@ impl UniversalMarker {
/// This should only be used when evaluating a marker that is known not to /// This should only be used when evaluating a marker that is known not to
/// have any extras. For example, the PEP 508 markers on a fork. /// have any extras. For example, the PEP 508 markers on a fork.
pub(crate) fn evaluate_no_extras(self, env: &MarkerEnvironment) -> bool { pub(crate) fn evaluate_no_extras(self, env: &MarkerEnvironment) -> bool {
self.marker.evaluate(env, &[]) self.marker.evaluate(env, &MarkerVariantsUniversal, &[])
} }
/// Returns true if this universal marker is satisfied by the given marker /// Returns true if this universal marker is satisfied by the given marker
@ -305,6 +308,7 @@ impl UniversalMarker {
pub(crate) fn evaluate<P, E, G>( pub(crate) fn evaluate<P, E, G>(
self, self,
env: &MarkerEnvironment, env: &MarkerEnvironment,
variants: &impl MarkerVariantsEnvironment,
projects: impl Iterator<Item = P>, projects: impl Iterator<Item = P>,
extras: impl Iterator<Item = (P, E)>, extras: impl Iterator<Item = (P, E)>,
groups: impl Iterator<Item = (P, G)>, groups: impl Iterator<Item = (P, G)>,
@ -321,6 +325,7 @@ impl UniversalMarker {
groups.map(|(package, group)| encode_package_group(package.borrow(), group.borrow())); groups.map(|(package, group)| encode_package_group(package.borrow(), group.borrow()));
self.marker.evaluate( self.marker.evaluate(
env, env,
variants,
&projects &projects
.chain(extras) .chain(extras)
.chain(groups) .chain(groups)
@ -829,7 +834,6 @@ pub(crate) fn resolve_conflicts(
mod tests { mod tests {
use super::*; use super::*;
use std::str::FromStr; use std::str::FromStr;
use uv_pypi_types::ConflictSet; use uv_pypi_types::ConflictSet;
/// Creates a collection of declared conflicts from the sets /// Creates a collection of declared conflicts from the sets
@ -959,7 +963,7 @@ mod tests {
.collect::<Vec<(PackageName, ExtraName)>>(); .collect::<Vec<(PackageName, ExtraName)>>();
let groups = Vec::<(PackageName, GroupName)>::new(); let groups = Vec::<(PackageName, GroupName)>::new();
assert!( assert!(
!UniversalMarker::new(MarkerTree::TRUE, cm).evaluate_only_extras(&extras, &groups), !UniversalMarker::new(MarkerTree::TRUE, cm).evaluate_only_extras(&extras, &groups,),
"expected `{extra_names:?}` to evaluate to `false` in `{cm:?}`" "expected `{extra_names:?}` to evaluate to `false` in `{cm:?}`"
); );
} }
@ -982,7 +986,7 @@ mod tests {
.collect::<Vec<(PackageName, ExtraName)>>(); .collect::<Vec<(PackageName, ExtraName)>>();
let groups = Vec::<(PackageName, GroupName)>::new(); let groups = Vec::<(PackageName, GroupName)>::new();
assert!( assert!(
UniversalMarker::new(MarkerTree::TRUE, cm).evaluate_only_extras(&extras, &groups), UniversalMarker::new(MarkerTree::TRUE, cm).evaluate_only_extras(&extras, &groups,),
"expected `{extra_names:?}` to evaluate to `true` in `{cm:?}`" "expected `{extra_names:?}` to evaluate to `true` in `{cm:?}`"
); );
} }

View File

@ -11,15 +11,16 @@ use uv_client::{FlatIndexEntry, OwnedArchive, SimpleDetailMetadata, VersionFiles
use uv_configuration::BuildOptions; use uv_configuration::BuildOptions;
use uv_distribution_filename::{DistFilename, WheelFilename}; use uv_distribution_filename::{DistFilename, WheelFilename};
use uv_distribution_types::{ use uv_distribution_types::{
HashComparison, IncompatibleSource, IncompatibleWheel, IndexUrl, PrioritizedDist, HashComparison, IncompatibleSource, IncompatibleWheel, IndexEntryFilename, IndexUrl,
RegistryBuiltWheel, RegistrySourceDist, RequiresPython, SourceDistCompatibility, PrioritizedDist, RegistryBuiltWheel, RegistrySourceDist, RegistryVariantsJson, RequiresPython,
WheelCompatibility, SourceDistCompatibility, WheelCompatibility,
}; };
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_pep440::Version; use uv_pep440::Version;
use uv_platform_tags::{IncompatibleTag, TagCompatibility, Tags}; use uv_platform_tags::{IncompatibleTag, TagCompatibility, Tags};
use uv_pypi_types::{HashDigest, ResolutionMetadata, Yanked}; use uv_pypi_types::{HashDigest, ResolutionMetadata, Yanked};
use uv_types::HashStrategy; use uv_types::HashStrategy;
use uv_variants::VariantPriority;
use uv_warnings::warn_user_once; use uv_warnings::warn_user_once;
use crate::flat_index::FlatDistributions; use crate::flat_index::FlatDistributions;
@ -467,7 +468,7 @@ impl VersionMapLazy {
let yanked = file.yanked.as_deref(); let yanked = file.yanked.as_deref();
let hashes = file.hashes.clone(); let hashes = file.hashes.clone();
match filename { match filename {
DistFilename::WheelFilename(filename) => { IndexEntryFilename::DistFilename(DistFilename::WheelFilename(filename)) => {
let compatibility = self.wheel_compatibility( let compatibility = self.wheel_compatibility(
&filename, &filename,
&filename.name, &filename.name,
@ -484,7 +485,9 @@ impl VersionMapLazy {
}; };
priority_dist.insert_built(dist, hashes, compatibility); priority_dist.insert_built(dist, hashes, compatibility);
} }
DistFilename::SourceDistFilename(filename) => { IndexEntryFilename::DistFilename(DistFilename::SourceDistFilename(
filename,
)) => {
let compatibility = self.source_dist_compatibility( let compatibility = self.source_dist_compatibility(
&filename.name, &filename.name,
&filename.version, &filename.version,
@ -503,6 +506,14 @@ impl VersionMapLazy {
}; };
priority_dist.insert_source(dist, hashes, compatibility); priority_dist.insert_source(dist, hashes, compatibility);
} }
IndexEntryFilename::VariantJson(filename) => {
let variant_json = RegistryVariantsJson {
filename,
file: Box::new(file),
index: self.index.clone(),
};
priority_dist.insert_variant_json(variant_json);
}
} }
} }
if priority_dist.is_empty() { if priority_dist.is_empty() {
@ -589,8 +600,8 @@ impl VersionMapLazy {
} }
} }
// Determine a compatibility for the wheel based on tags. // Determine a priority for the wheel based on tags.
let priority = if let Some(tags) = &self.tags { let tag_priority = if let Some(tags) = &self.tags {
match filename.compatibility(tags) { match filename.compatibility(tags) {
TagCompatibility::Incompatible(tag) => { TagCompatibility::Incompatible(tag) => {
return WheelCompatibility::Incompatible(IncompatibleWheel::Tag(tag)); return WheelCompatibility::Incompatible(IncompatibleWheel::Tag(tag));
@ -608,6 +619,13 @@ impl VersionMapLazy {
None None
}; };
// TODO(konsti): Currently we ignore variants here on only determine them later
let variant_priority = if filename.variant().is_none() {
VariantPriority::NonVariant
} else {
VariantPriority::Unknown
};
// Check if hashes line up. If hashes aren't required, they're considered matching. // Check if hashes line up. If hashes aren't required, they're considered matching.
let hash_policy = self.hasher.get_package(name, version); let hash_policy = self.hasher.get_package(name, version);
let required_hashes = hash_policy.digests(); let required_hashes = hash_policy.digests();
@ -626,7 +644,12 @@ impl VersionMapLazy {
// Break ties with the build tag. // Break ties with the build tag.
let build_tag = filename.build_tag().cloned(); let build_tag = filename.build_tag().cloned();
WheelCompatibility::Compatible(hash, priority, build_tag) WheelCompatibility::Compatible {
hash,
tag_priority,
variant_priority,
build_tag,
}
} }
} }

View File

@ -3,6 +3,8 @@ use std::cmp::PartialEq;
use std::ops::Deref; use std::ops::Deref;
/// An optimized type for immutable identifiers. Represented as an [`arcstr::ArcStr`] internally. /// An optimized type for immutable identifiers. Represented as an [`arcstr::ArcStr`] internally.
///
/// This type is one pointer wide.
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct SmallString(arcstr::ArcStr); pub struct SmallString(arcstr::ArcStr);
@ -159,3 +161,13 @@ impl schemars::JsonSchema for SmallString {
String::json_schema(generator) String::json_schema(generator)
} }
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn small_str_size() {
assert_eq!(size_of::<SmallString>(), size_of::<usize>());
}
}

View File

@ -1236,4 +1236,15 @@ impl EnvVars {
/// around invalid artifacts in rare cases. /// around invalid artifacts in rare cases.
#[attr_added_in("0.8.23")] #[attr_added_in("0.8.23")]
pub const UV_SKIP_WHEEL_FILENAME_CHECK: &'static str = "UV_SKIP_WHEEL_FILENAME_CHECK"; pub const UV_SKIP_WHEEL_FILENAME_CHECK: &'static str = "UV_SKIP_WHEEL_FILENAME_CHECK";
/// A comma separated list of variant provider backends that use the current environment instead
/// of an isolated environment.
///
/// The requirements need to be installed in the current environment, no provider `requires`
/// will be installed. This option is intended for development and as an escape hatch where
/// isolation fails.
///
/// Example: `UV_NO_PROVIDER_ISOLATION=gpu.provider:api,cpu,blas.backend`
#[attr_added_in("0.9.2")]
pub const UV_NO_PROVIDER_ISOLATION: &'static str = "UV_NO_PROVIDER_ISOLATION";
} }

View File

@ -24,9 +24,11 @@ uv-git = { workspace = true }
uv-normalize = { workspace = true } uv-normalize = { workspace = true }
uv-once-map = { workspace = true } uv-once-map = { workspace = true }
uv-pep440 = { workspace = true } uv-pep440 = { workspace = true }
uv-pep508 = { workspace = true }
uv-pypi-types = { workspace = true } uv-pypi-types = { workspace = true }
uv-python = { workspace = true } uv-python = { workspace = true }
uv-redacted = { workspace = true } uv-redacted = { workspace = true }
uv-variants = { workspace = true }
uv-workspace = { workspace = true } uv-workspace = { workspace = true }
anyhow = { workspace = true } anyhow = { workspace = true }

View File

@ -10,6 +10,7 @@ use uv_distribution_types::{
}; };
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_pep440::Version; use uv_pep440::Version;
use uv_pep508::MarkerVariantsUniversal;
use uv_pypi_types::{HashDigest, HashDigests, HashError, ResolverMarkerEnvironment}; use uv_pypi_types::{HashDigest, HashDigests, HashError, ResolverMarkerEnvironment};
use uv_redacted::DisplaySafeUrl; use uv_redacted::DisplaySafeUrl;
@ -134,9 +135,11 @@ impl HashStrategy {
// First, index the constraints by name. // First, index the constraints by name.
for (requirement, digests) in constraints { for (requirement, digests) in constraints {
if !requirement if !requirement.evaluate_markers(
.evaluate_markers(marker_env.map(ResolverMarkerEnvironment::markers), &[]) marker_env.map(ResolverMarkerEnvironment::markers),
{ &MarkerVariantsUniversal,
&[],
) {
continue; continue;
} }
@ -178,9 +181,11 @@ impl HashStrategy {
// package. // package.
let mut requirement_hashes = FxHashMap::<VersionId, Vec<HashDigest>>::default(); let mut requirement_hashes = FxHashMap::<VersionId, Vec<HashDigest>>::default();
for (requirement, digests) in requirements { for (requirement, digests) in requirements {
if !requirement if !requirement.evaluate_markers(
.evaluate_markers(marker_env.map(ResolverMarkerEnvironment::markers), &[]) marker_env.map(ResolverMarkerEnvironment::markers),
{ &MarkerVariantsUniversal,
&[],
) {
continue; continue;
} }

View File

@ -5,7 +5,6 @@ use std::path::{Path, PathBuf};
use anyhow::Result; use anyhow::Result;
use rustc_hash::FxHashSet; use rustc_hash::FxHashSet;
use uv_cache::Cache; use uv_cache::Cache;
use uv_configuration::{BuildKind, BuildOptions, BuildOutput, SourceStrategy}; use uv_configuration::{BuildKind, BuildOptions, BuildOutput, SourceStrategy};
use uv_distribution_filename::DistFilename; use uv_distribution_filename::DistFilename;
@ -17,6 +16,8 @@ use uv_distribution_types::{
use uv_git::GitResolver; use uv_git::GitResolver;
use uv_normalize::PackageName; use uv_normalize::PackageName;
use uv_python::{Interpreter, PythonEnvironment}; use uv_python::{Interpreter, PythonEnvironment};
use uv_variants::VariantProviderOutput;
use uv_variants::cache::VariantProviderCache;
use uv_workspace::WorkspaceCache; use uv_workspace::WorkspaceCache;
use crate::{BuildArena, BuildIsolation}; use crate::{BuildArena, BuildIsolation};
@ -60,6 +61,7 @@ use crate::{BuildArena, BuildIsolation};
/// them. /// them.
pub trait BuildContext { pub trait BuildContext {
type SourceDistBuilder: SourceBuildTrait; type SourceDistBuilder: SourceBuildTrait;
type VariantsBuilder: VariantsTrait;
// Note: this function is async deliberately, because downstream code may need to // Note: this function is async deliberately, because downstream code may need to
// run async code to get the interpreter, to resolve the Python version. // run async code to get the interpreter, to resolve the Python version.
@ -72,6 +74,9 @@ pub trait BuildContext {
/// Return a reference to the Git resolver. /// Return a reference to the Git resolver.
fn git(&self) -> &GitResolver; fn git(&self) -> &GitResolver;
/// Return a reference to the variant cache.
fn variants(&self) -> &VariantProviderCache;
/// Return a reference to the build arena. /// Return a reference to the build arena.
fn build_arena(&self) -> &BuildArena<Self::SourceDistBuilder>; fn build_arena(&self) -> &BuildArena<Self::SourceDistBuilder>;
@ -161,6 +166,14 @@ pub trait BuildContext {
build_kind: BuildKind, build_kind: BuildKind,
version_id: Option<&'a str>, version_id: Option<&'a str>,
) -> impl Future<Output = Result<Option<DistFilename>, impl IsBuildBackendError>> + 'a; ) -> impl Future<Output = Result<Option<DistFilename>, impl IsBuildBackendError>> + 'a;
/// Set up the variants for the given provider.
fn setup_variants<'a>(
&'a self,
backend_name: String,
provider: &'a uv_variants::variants_json::Provider,
build_output: BuildOutput,
) -> impl Future<Output = Result<Self::VariantsBuilder, anyhow::Error>> + 'a;
} }
/// A wrapper for `uv_build::SourceBuild` to avoid cyclical crate dependencies. /// A wrapper for `uv_build::SourceBuild` to avoid cyclical crate dependencies.
@ -189,6 +202,10 @@ pub trait SourceBuildTrait {
) -> impl Future<Output = Result<String, AnyErrorBuild>> + 'a; ) -> impl Future<Output = Result<String, AnyErrorBuild>> + 'a;
} }
pub trait VariantsTrait {
fn query(&self) -> impl Future<Output = Result<VariantProviderOutput>>;
}
/// A wrapper for [`uv_installer::SitePackages`] /// A wrapper for [`uv_installer::SitePackages`]
pub trait InstalledPackagesProvider: Clone + Send + Sync + 'static { pub trait InstalledPackagesProvider: Clone + Send + Sync + 'static {
fn iter(&self) -> impl Iterator<Item = &InstalledDist>; fn iter(&self) -> impl Iterator<Item = &InstalledDist>;

View File

@ -0,0 +1,36 @@
[package]
name = "uv-variant-frontend"
version = "0.0.3"
description = "This is an internal component crate of uv"
edition = { workspace = true }
rust-version = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
authors = { workspace = true }
license = { workspace = true }
[lints]
workspace = true
[dependencies]
uv-configuration = { workspace = true }
uv-distribution-types = { workspace = true }
uv-fs = { workspace = true }
uv-preview = { workspace = true }
uv-python = { workspace = true }
uv-static = { workspace = true }
uv-types = { workspace = true }
uv-virtualenv = { workspace = true }
uv-variants = { workspace = true }
anstream = { workspace = true }
anyhow = { workspace = true }
fs-err = { workspace = true }
indoc = { workspace = true }
owo-colors = { workspace = true }
rustc-hash = { workspace = true }
serde_json = { workspace = true, features = ["preserve_order"] }
tempfile = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }

View File

@ -0,0 +1,109 @@
use std::env;
use std::fmt::{Display, Formatter};
use std::io;
use std::path::PathBuf;
use std::process::ExitStatus;
use owo_colors::OwoColorize;
use thiserror::Error;
use tracing::error;
use uv_configuration::BuildOutput;
use uv_types::AnyErrorBuild;
use crate::PythonRunnerOutput;
#[derive(Error, Debug)]
pub enum Error {
#[error(transparent)]
Io(#[from] io::Error),
#[error("Failed to resolve requirements from {0}")]
RequirementsResolve(&'static str, #[source] AnyErrorBuild),
#[error("Failed to install requirements from {0}")]
RequirementsInstall(&'static str, #[source] AnyErrorBuild),
#[error("Failed to create temporary virtualenv")]
Virtualenv(#[from] uv_virtualenv::Error),
#[error("Provider plugin `{0}` is missing `requires`")]
MissingRequires(String),
#[error("Failed to run `{0}`")]
CommandFailed(PathBuf, #[source] io::Error),
#[error("The build backend returned an error")]
ProviderBackend(#[from] ProviderBackendError),
#[error("Failed to build PATH for build script")]
BuildScriptPath(#[source] env::JoinPathsError),
#[error(transparent)]
SerdeJson(#[from] serde_json::Error),
#[error(
"There must be exactly one `requires` value matching the current environment for \
{backend_name}, but there are {matching}"
)]
InvalidRequires {
backend_name: String,
matching: usize,
},
}
#[derive(Debug, Error)]
pub struct ProviderBackendError {
message: String,
exit_code: ExitStatus,
stdout: Vec<String>,
stderr: Vec<String>,
}
impl Display for ProviderBackendError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{} ({})", self.message, self.exit_code)?;
let mut non_empty = false;
if self.stdout.iter().any(|line| !line.trim().is_empty()) {
write!(f, "\n\n{}\n{}", "[stdout]".red(), self.stdout.join("\n"))?;
non_empty = true;
}
if self.stderr.iter().any(|line| !line.trim().is_empty()) {
write!(f, "\n\n{}\n{}", "[stderr]".red(), self.stderr.join("\n"))?;
non_empty = true;
}
if non_empty {
writeln!(f)?;
}
write!(
f,
"\n{}{} This usually indicates a problem with the package or the build environment.",
"hint".bold().cyan(),
":".bold()
)?;
Ok(())
}
}
impl Error {
/// Construct an [`Error`] from the output of a failed command.
pub(crate) fn from_command_output(
message: String,
output: &PythonRunnerOutput,
level: BuildOutput,
) -> Self {
match level {
BuildOutput::Stderr | BuildOutput::Quiet => {
Self::ProviderBackend(ProviderBackendError {
message,
exit_code: output.status,
stdout: vec![],
stderr: vec![],
})
}
BuildOutput::Debug => Self::ProviderBackend(ProviderBackendError {
message,
exit_code: output.status,
stdout: output.stdout.clone(),
stderr: output.stderr.clone(),
}),
}
}
}

View File

@ -0,0 +1,441 @@
//! Detect compatible variants from a variant provider.
mod error;
use std::borrow::Cow;
use std::ffi::OsString;
use std::fmt::Write;
use std::io;
use std::path::{Path, PathBuf};
use std::process::ExitStatus;
use std::{env, iter};
use fs_err as fs;
use indoc::formatdoc;
use rustc_hash::FxHashMap;
use tempfile::TempDir;
use tokio::io::AsyncBufReadExt;
use tokio::process::Command;
use tokio::sync::Semaphore;
use tracing::{Instrument, debug, info_span};
pub use crate::error::Error;
use uv_configuration::BuildOutput;
use uv_distribution_types::Requirement;
use uv_fs::{PythonExt, Simplified};
use uv_preview::Preview;
use uv_python::{Interpreter, PythonEnvironment};
use uv_static::EnvVars;
use uv_types::{BuildContext, BuildStack, VariantsTrait};
use uv_variants::VariantProviderOutput;
use uv_variants::variants_json::Provider;
use uv_virtualenv::OnExisting;
pub struct VariantBuild {
temp_dir: TempDir,
/// The backend to use.
backend_name: String,
/// The backend to use.
backend: Provider,
/// The virtual environment in which to build the source distribution.
venv: PythonEnvironment,
/// Whether to send build output to `stderr` or `tracing`, etc.
level: BuildOutput,
/// Modified PATH that contains the `venv_bin`, `user_path` and `system_path` variables in that
/// order.
modified_path: OsString,
/// Environment variables to be passed in.
environment_variables: FxHashMap<OsString, OsString>,
/// Runner for Python scripts.
runner: PythonRunner,
}
impl VariantsTrait for VariantBuild {
async fn query(&self) -> anyhow::Result<VariantProviderOutput> {
Ok(self.build().await?)
}
}
impl VariantBuild {
/// Create a virtual environment in which to run a variant provider.
pub async fn setup(
backend_name: String,
backend: &Provider,
interpreter: &Interpreter,
build_context: &impl BuildContext,
mut environment_variables: FxHashMap<OsString, OsString>,
level: BuildOutput,
concurrent_builds: usize,
) -> Result<Self, Error> {
let temp_dir = build_context.cache().venv_dir()?;
// TODO(konsti): This is not the right location to parse the env var.
let plugin_api = Self::plugin_api(&backend_name, backend, interpreter)?;
let no_isolation =
env::var(EnvVars::UV_NO_PROVIDER_ISOLATION).is_ok_and(|no_provider_isolation| {
no_provider_isolation
.split(',')
.any(|api| (api) == plugin_api)
});
// TODO(konsti): Integrate this properly with the configuration system.
if no_isolation {
debug!("Querying provider plugin without isolation: {backend_name}");
let runner = PythonRunner::new(concurrent_builds, level);
let env = PythonEnvironment::from_interpreter(interpreter.clone());
// The unmodified path
let modified_path = OsString::from(env.scripts());
return Ok(Self {
temp_dir,
backend_name,
backend: backend.clone(),
venv: env,
level,
modified_path,
environment_variables,
runner,
});
}
// Create a virtual environment.
let venv = uv_virtualenv::create_venv(
temp_dir.path(),
interpreter.clone(),
uv_virtualenv::Prompt::None,
false,
// This is a fresh temp dir
OnExisting::Fail,
false,
false,
false,
Preview::default(), // TODO(konsti)
)?;
// Resolve and install the provider requirements.
let requirements = backend
.requires
.as_ref()
.ok_or_else(|| Error::MissingRequires(backend_name.clone()))?
.iter()
.cloned()
.map(Requirement::from)
.collect::<Vec<_>>();
let resolved_requirements = build_context
.resolve(&requirements, &BuildStack::empty())
.await
.map_err(|err| {
Error::RequirementsResolve("`variant.providers.requires`", err.into())
})?;
build_context
.install(&resolved_requirements, &venv, &BuildStack::empty())
.await
.map_err(|err| {
Error::RequirementsInstall("`variant.providers.requires`", err.into())
})?;
// Figure out what the modified path should be, and remove the PATH variable from the
// environment variables if it's there.
let user_path = environment_variables.remove(&OsString::from(EnvVars::PATH));
// See if there is an OS PATH variable.
let os_path = env::var_os(EnvVars::PATH);
// Prepend the user supplied PATH to the existing OS PATH.
let modified_path = if let Some(user_path) = user_path {
match os_path {
// Prepend the user supplied PATH to the existing PATH.
Some(env_path) => {
let user_path = PathBuf::from(user_path);
let new_path = env::split_paths(&user_path).chain(env::split_paths(&env_path));
Some(env::join_paths(new_path).map_err(Error::BuildScriptPath)?)
}
// Use the user supplied PATH.
None => Some(user_path),
}
} else {
os_path
};
// Prepend the venv bin directory to the modified path.
let modified_path = if let Some(path) = modified_path {
let venv_path = iter::once(venv.scripts().to_path_buf()).chain(env::split_paths(&path));
env::join_paths(venv_path).map_err(Error::BuildScriptPath)?
} else {
OsString::from(venv.scripts())
};
let runner = PythonRunner::new(concurrent_builds, level);
Ok(Self {
temp_dir,
backend_name,
backend: backend.clone(),
venv,
level,
modified_path,
environment_variables,
runner,
})
}
// Not a method to be callable in the constructor.
pub fn plugin_api<'a>(
backend_name: &str,
backend: &'a Provider,
interpreter: &Interpreter,
) -> Result<Cow<'a, str>, Error> {
if let Some(plugin_api) = &backend.plugin_api {
Ok(Cow::Borrowed(plugin_api))
} else {
let requires = backend
.requires
.as_ref()
.ok_or_else(|| Error::MissingRequires(backend_name.to_string()))?
.iter()
.filter(|requirement| {
requirement.evaluate_markers(interpreter.markers(), &Vec::new().as_slice(), &[])
})
.collect::<Vec<_>>();
if let [requires] = requires.as_slice() {
Ok(requires.name.as_dist_info_name())
} else {
Err(Error::InvalidRequires {
backend_name: backend_name.to_string(),
matching: requires.len(),
})
}
}
}
pub fn import(&self) -> Result<String, Error> {
let plugin_api =
Self::plugin_api(&self.backend_name, &self.backend, self.venv.interpreter())?;
let import = if let Some((path, object)) = plugin_api.split_once(':') {
format!("from {path} import {object} as backend")
} else {
format!("import {plugin_api} as backend")
};
Ok(formatdoc! {r#"
import sys
if sys.path[0] == "":
sys.path.pop(0)
{import}
if callable(backend):
backend = backend()
"#})
}
/// Run a variant provider to infer compatible variants.
pub async fn build(&self) -> Result<VariantProviderOutput, Error> {
// Write the hook output to a file so that we can read it back reliably.
let out_file = self.temp_dir.path().join("output.json");
// Construct the appropriate build script based on the build kind.
let script = formatdoc! {
r#"
{backend}
import json
configs = backend.get_supported_configs()
features = {{config.name: config.values for config in configs}}
output = {{"namespace": backend.namespace, "features": features}}
with open("{out_file}", "w") as fp:
fp.write(json.dumps(output))
"#,
backend = self.import()?,
out_file = out_file.escape_for_python()
};
let span = info_span!(
"run_variant_provider_script",
backend_name = self.backend_name
);
let output = self
.runner
.run_script(
&self.venv,
&script,
self.temp_dir.path(),
&self.environment_variables,
&self.modified_path,
)
.instrument(span)
.await?;
if !output.status.success() {
return Err(Error::from_command_output(
format!(
"Call to variant backend failed in `{}`",
self.backend
.plugin_api
.as_deref()
.unwrap_or(&self.backend_name)
),
&output,
self.level,
));
}
// Read as JSON.
let json = fs::read(&out_file).map_err(|err| {
Error::CommandFailed(self.venv.python_executable().to_path_buf(), err)
})?;
let output = serde_json::from_slice::<VariantProviderOutput>(&json).map_err(|err| {
Error::CommandFailed(self.venv.python_executable().to_path_buf(), err.into())
})?;
Ok(output)
}
}
/// A runner that manages the execution of external python processes with a
/// concurrency limit.
#[derive(Debug)]
struct PythonRunner {
control: Semaphore,
level: BuildOutput,
}
#[derive(Debug)]
struct PythonRunnerOutput {
stdout: Vec<String>,
stderr: Vec<String>,
status: ExitStatus,
}
impl PythonRunner {
/// Create a `PythonRunner` with the provided concurrency limit and output level.
fn new(concurrency: usize, level: BuildOutput) -> Self {
Self {
control: Semaphore::new(concurrency),
level,
}
}
/// Spawn a process that runs a python script in the provided environment.
///
/// If the concurrency limit has been reached this method will wait until a pending
/// script completes before spawning this one.
///
/// Note: It is the caller's responsibility to create an informative span.
async fn run_script(
&self,
venv: &PythonEnvironment,
script: &str,
source_tree: &Path,
environment_variables: &FxHashMap<OsString, OsString>,
modified_path: &OsString,
) -> Result<PythonRunnerOutput, Error> {
/// Read lines from a reader and store them in a buffer.
async fn read_from(
mut reader: tokio::io::Split<tokio::io::BufReader<impl tokio::io::AsyncRead + Unpin>>,
mut printer: Printer,
buffer: &mut Vec<String>,
) -> io::Result<()> {
loop {
match reader.next_segment().await? {
Some(line_buf) => {
let line_buf = line_buf.strip_suffix(b"\r").unwrap_or(&line_buf);
let line = String::from_utf8_lossy(line_buf).into();
let _ = write!(printer, "{line}");
buffer.push(line);
}
None => return Ok(()),
}
}
}
let _permit = self.control.acquire().await.unwrap();
let mut child = Command::new(venv.python_executable())
.args(["-c", script])
.current_dir(source_tree.simplified())
.envs(environment_variables)
.env(EnvVars::PATH, modified_path)
.env(EnvVars::VIRTUAL_ENV, venv.root())
.env(EnvVars::CLICOLOR_FORCE, "1")
.env(EnvVars::PYTHONIOENCODING, "utf-8:backslashreplace")
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.map_err(|err| Error::CommandFailed(venv.python_executable().to_path_buf(), err))?;
// Create buffers to capture `stdout` and `stderr`.
let mut stdout_buf = Vec::with_capacity(1024);
let mut stderr_buf = Vec::with_capacity(1024);
// Create separate readers for `stdout` and `stderr`.
let stdout_reader = tokio::io::BufReader::new(child.stdout.take().unwrap()).split(b'\n');
let stderr_reader = tokio::io::BufReader::new(child.stderr.take().unwrap()).split(b'\n');
// Asynchronously read from the in-memory pipes.
let printer = Printer::from(self.level);
let result = tokio::join!(
read_from(stdout_reader, printer, &mut stdout_buf),
read_from(stderr_reader, printer, &mut stderr_buf),
);
match result {
(Ok(()), Ok(())) => {}
(Err(err), _) | (_, Err(err)) => {
return Err(Error::CommandFailed(
venv.python_executable().to_path_buf(),
err,
));
}
}
// Wait for the child process to finish.
let status = child
.wait()
.await
.map_err(|err| Error::CommandFailed(venv.python_executable().to_path_buf(), err))?;
Ok(PythonRunnerOutput {
stdout: stdout_buf,
stderr: stderr_buf,
status,
})
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum Printer {
/// Send the provider output to `stderr`.
Stderr,
/// Send the provider output to `tracing`.
Debug,
/// Hide the provider output.
Quiet,
}
impl From<BuildOutput> for Printer {
fn from(output: BuildOutput) -> Self {
match output {
BuildOutput::Stderr => Self::Stderr,
BuildOutput::Debug => Self::Debug,
BuildOutput::Quiet => Self::Quiet,
}
}
}
impl Write for Printer {
fn write_str(&mut self, s: &str) -> std::fmt::Result {
match self {
Self::Stderr => {
anstream::eprintln!("{s}");
}
Self::Debug => {
debug!("{s}");
}
Self::Quiet => {}
}
Ok(())
}
}

View File

@ -0,0 +1,32 @@
[package]
name = "uv-variants"
version = "0.0.3"
description = "This is an internal component crate of uv"
edition = { workspace = true }
rust-version = { workspace = true }
homepage = { workspace = true }
repository = { workspace = true }
authors = { workspace = true }
license = { workspace = true }
[dependencies]
uv-distribution-filename = { workspace = true }
uv-normalize = { workspace = true }
uv-once-map = { workspace = true }
uv-pep440 = { workspace = true, features = ["serde"] }
uv-pep508 = { workspace = true, features = ["serde"] }
uv-pypi-types = { workspace = true }
indexmap = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
[dev-dependencies]
insta = "1.43.1"
itertools = { workspace = true }
serde_json = { workspace = true, features = ["preserve_order"] }
[lints]
workspace = true

View File

@ -0,0 +1,23 @@
use std::hash::BuildHasherDefault;
use std::sync::Arc;
use rustc_hash::FxHasher;
use uv_once_map::OnceMap;
use crate::VariantProviderOutput;
use crate::variants_json::Provider;
type FxOnceMap<K, V> = OnceMap<K, V, BuildHasherDefault<FxHasher>>;
/// An in-memory cache for variant provider outputs.
#[derive(Default)]
pub struct VariantProviderCache(FxOnceMap<Provider, Arc<VariantProviderOutput>>);
impl std::ops::Deref for VariantProviderCache {
type Target = FxOnceMap<Provider, Arc<VariantProviderOutput>>;
fn deref(&self) -> &Self::Target {
&self.0
}
}

View File

@ -0,0 +1,36 @@
use indexmap::IndexMap;
use uv_pep508::{VariantFeature, VariantNamespace, VariantValue};
pub mod cache;
pub mod resolved_variants;
pub mod variant_lock;
pub mod variant_with_label;
pub mod variants_json;
/// Wire format between with the Python shim for provider plugins.
#[derive(Debug, Clone, Eq, PartialEq, serde::Deserialize)]
pub struct VariantProviderOutput {
/// The namespace of the provider.
pub namespace: VariantNamespace,
/// Features (in order) mapped to their properties (in order).
pub features: IndexMap<VariantFeature, Vec<VariantValue>>,
}
/// The priority of a variant.
///
/// When we first create a `PrioritizedDist`, there is no information about which variants are
/// supported yet. In universal resolution, we don't need to query this information. In platform
/// specific resolution, we determine a best variant for the current platform - if any - after
/// selecting a version.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum VariantPriority {
/// A variant wheel, it's unclear whether it's compatible.
///
/// Variants only become supported after we ran the provider plugins.
Unknown,
/// A non-variant wheel.
NonVariant,
/// The supported variant wheel in this prioritized dist with the highest score.
BestVariant,
}

View File

@ -0,0 +1,295 @@
use std::sync::Arc;
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::{debug, trace, warn};
use uv_distribution_filename::VariantLabel;
use uv_pep508::{VariantNamespace, VariantValue};
use crate::VariantProviderOutput;
use crate::variants_json::{DefaultPriorities, Variant, VariantsJsonContent};
#[derive(Debug, Clone)]
pub struct ResolvedVariants {
pub variants_json: VariantsJsonContent,
pub resolved_namespaces: FxHashMap<VariantNamespace, Arc<VariantProviderOutput>>,
/// Namespaces where `enable-if` didn't match.
pub disabled_namespaces: FxHashSet<VariantNamespace>,
}
impl ResolvedVariants {
pub fn score_variant(&self, variant: &VariantLabel) -> Option<Vec<usize>> {
let Some(variants_properties) = self.variants_json.variants.get(variant) else {
warn!("Variant {variant} is missing in variants.json");
return None;
};
score_variant(
&self.variants_json.default_priorities,
&self.resolved_namespaces,
&self.disabled_namespaces,
variants_properties,
)
}
}
/// Return a priority score for the variant (higher is better) or `None` if it isn't compatible.
pub fn score_variant(
default_priorities: &DefaultPriorities,
target_namespaces: &FxHashMap<VariantNamespace, Arc<VariantProviderOutput>>,
disabled_namespaces: &FxHashSet<VariantNamespace>,
variants_properties: &Variant,
) -> Option<Vec<usize>> {
for (namespace, features) in &**variants_properties {
for (feature, properties) in features {
let resolved_properties = target_namespaces
.get(namespace)
.and_then(|namespace| namespace.features.get(feature))?;
if !properties
.iter()
.any(|property| resolved_properties.contains(property))
{
return None;
}
}
}
// TODO(konsti): This is performance sensitive, prepare priorities and use a pairwise wheel
// comparison function instead.
let mut scores = Vec::new();
for namespace in &default_priorities.namespace {
if disabled_namespaces.contains(namespace) {
trace!("Skipping disabled namespace: {}", namespace);
continue;
}
// Explicit priorities are optional, but take priority over the provider
let explicit_feature_priorities = default_priorities.feature.get(namespace);
let Some(target_variants) = target_namespaces.get(namespace) else {
// TODO(konsti): Can this even happen?
debug!("Missing namespace priority: {namespace}");
continue;
};
let feature_priorities = explicit_feature_priorities.into_iter().flatten().chain(
target_variants.features.keys().filter(|priority| {
explicit_feature_priorities.is_none_or(|explicit| !explicit.contains(priority))
}),
);
for feature in feature_priorities {
let value_priorities: Vec<VariantValue> = default_priorities
.property
.get(namespace)
.and_then(|namespace_features| namespace_features.get(feature))
.into_iter()
.flatten()
.cloned()
.chain(
target_namespaces
.get(namespace)
.and_then(|namespace| namespace.features.get(feature).cloned())
.into_iter()
.flatten(),
)
.collect();
let Some(wheel_properties) = variants_properties
.get(namespace)
.and_then(|namespace| namespace.get(feature))
else {
scores.push(0);
continue;
};
// Determine the highest scoring property
// Reversed to give a higher score to earlier entries
let score = value_priorities.len()
- value_priorities
.iter()
.position(|feature| wheel_properties.contains(feature))
.unwrap_or(value_priorities.len());
scores.push(score);
}
}
Some(scores)
}
#[cfg(test)]
mod tests {
use insta::assert_snapshot;
use itertools::Itertools;
use rustc_hash::{FxHashMap, FxHashSet};
use serde_json::json;
use std::sync::Arc;
use uv_pep508::VariantNamespace;
use crate::VariantProviderOutput;
use crate::resolved_variants::score_variant;
use crate::variants_json::{DefaultPriorities, Variant};
fn host() -> FxHashMap<VariantNamespace, Arc<VariantProviderOutput>> {
serde_json::from_value(json!({
"gpu": {
"namespace": "gpu",
"features": {
// Even though they are ahead of CUDA here, they are sorted below it due to the
// default priorities
"rocm": ["rocm68"],
"xpu": ["xpu1"],
"cuda": ["cu128", "cu126"]
}
},
"cpu": {
"namespace": "cpu",
"features": {
"level": ["x86_64_v2", "x86_64_v1"]
}
},
}))
.unwrap()
}
// Default priorities in `variants.json`
fn default_priorities() -> DefaultPriorities {
serde_json::from_value(json!({
"namespace": ["gpu", "cpu", "blas", "not_used_namespace"],
"feature": {
"gpu": ["cuda", "not_used_feature"],
"cpu": ["level"],
},
"property": {
"cpu": {
"level": ["x86_64_v4", "x86_64_v3", "x86_64_v2", "x86_64_v1", "not_used_value"],
},
},
}))
.unwrap()
}
fn score(variant: &Variant) -> Option<String> {
let score = score_variant(
&default_priorities(),
&host(),
&FxHashSet::default(),
variant,
)?;
Some(score.iter().map(ToString::to_string).join(", "))
}
#[test]
fn incompatible_variants() {
let incompatible_namespace: Variant = serde_json::from_value(json!({
"serial": {
"usb": ["usb3"],
},
}))
.unwrap();
assert_eq!(score(&incompatible_namespace), None);
let incompatible_feature: Variant = serde_json::from_value(json!({
"gpu": {
"rocm": ["rocm69"],
},
}))
.unwrap();
assert_eq!(score(&incompatible_feature), None);
let incompatible_value: Variant = serde_json::from_value(json!({
"gpu": {
"cuda": ["cu130"],
},
}))
.unwrap();
assert_eq!(score(&incompatible_value), None);
}
#[test]
fn variant_sorting() {
let cu128_v2: Variant = serde_json::from_value(json!({
"gpu": {
"cuda": ["cu128"],
},
"cpu": {
"level": ["x86_64_v2"],
},
}))
.unwrap();
let cu128_v1: Variant = serde_json::from_value(json!({
"gpu": {
"cuda": ["cu128"],
},
"cpu": {
"level": ["x86_64_v1"],
},
}))
.unwrap();
let cu126_v2: Variant = serde_json::from_value(json!({
"gpu": {
"cuda": ["cu126"],
},
"cpu": {
"level": ["x86_64_v2"],
},
}))
.unwrap();
let cu126_v1: Variant = serde_json::from_value(json!({
"gpu": {
"cuda": ["cu126"],
},
"cpu": {
"level": ["x86_64_v1"],
},
}))
.unwrap();
let rocm: Variant = serde_json::from_value(json!({
"gpu": {
"rocm": ["rocm68"],
},
}))
.unwrap();
let xpu: Variant = serde_json::from_value(json!({
"gpu": {
"xpu": ["xpu1"],
},
}))
.unwrap();
// If the namespace is missing, the variant is compatible but below the higher ranking
// namespace
let v1: Variant = serde_json::from_value(json!({
"cpu": {
"level": ["x86_64_v1"],
},
}))
.unwrap();
// The null variant is last.
let null: Variant = serde_json::from_value(json!({})).unwrap();
assert_snapshot!(score(&cu128_v2).unwrap(), @"2, 0, 0, 0, 5");
assert_snapshot!(score(&cu128_v1).unwrap(), @"2, 0, 0, 0, 4");
assert_snapshot!(score(&cu126_v2).unwrap(), @"1, 0, 0, 0, 5");
assert_snapshot!(score(&cu126_v1).unwrap(), @"1, 0, 0, 0, 4");
assert_snapshot!(score(&rocm).unwrap(), @"0, 0, 1, 0, 0");
assert_snapshot!(score(&xpu).unwrap(), @"0, 0, 0, 1, 0");
assert_snapshot!(score(&v1).unwrap(), @"0, 0, 0, 0, 4");
assert_snapshot!(score(&null).unwrap(), @"0, 0, 0, 0, 0");
let wheels = vec![
&cu128_v2, &cu128_v1, &cu126_v2, &cu126_v1, &rocm, &xpu, &v1, &null,
];
let mut wheels2 = wheels.clone();
// "shuffle"
wheels2.reverse();
wheels2.sort_by(|a, b| {
score_variant(&default_priorities(), &host(), &FxHashSet::default(), a)
.cmp(&score_variant(
&default_priorities(),
&host(),
&FxHashSet::default(),
b,
))
// higher is better
.reverse()
});
assert_eq!(wheels2, wheels);
}
}

View File

@ -0,0 +1,94 @@
use std::str::FromStr;
use indexmap::IndexMap;
use serde::{Deserialize, Deserializer};
use thiserror::Error;
use uv_normalize::PackageName;
use uv_pep440::Version;
use uv_pep508::{UnnamedRequirementUrl, VariantFeature, VariantNamespace, VariantValue};
use uv_pypi_types::VerbatimParsedUrl;
#[derive(Debug, Error)]
pub enum VariantLockError {
#[error("Invalid resolved requirement format: {0}")]
InvalidResolvedFormat(String),
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct VariantLock {
pub metadata: VariantLockMetadata,
pub provider: Vec<VariantLockProvider>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct VariantLockMetadata {
pub created_by: String,
pub version: Version,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct VariantLockProvider {
pub resolved: Vec<VariantLockResolved>,
pub plugin_api: Option<String>,
pub namespace: VariantNamespace,
pub properties: IndexMap<VariantFeature, Vec<VariantValue>>,
}
/// A resolved requirement in the form `<name>==<version>` or `<name> @ <url>`
#[derive(Debug, Clone)]
pub enum VariantLockResolved {
Version(PackageName, Version),
Url(PackageName, Box<VerbatimParsedUrl>),
}
impl VariantLockResolved {
pub fn name(&self) -> &PackageName {
match self {
Self::Version(name, _) => name,
Self::Url(name, _) => name,
}
}
}
impl FromStr for VariantLockResolved {
type Err = VariantLockError;
fn from_str(resolved: &str) -> Result<Self, Self::Err> {
if let Some((name, version)) = resolved.split_once("==") {
Ok(Self::Version(
PackageName::from_str(name.trim())
.map_err(|_| VariantLockError::InvalidResolvedFormat(resolved.to_string()))?,
Version::from_str(version.trim())
.map_err(|_| VariantLockError::InvalidResolvedFormat(resolved.to_string()))?,
))
} else if let Some((name, url)) = resolved.split_once(" @ ") {
Ok(Self::Url(
PackageName::from_str(name.trim())
.map_err(|_| VariantLockError::InvalidResolvedFormat(resolved.to_string()))?,
Box::new(
VerbatimParsedUrl::parse_unnamed_url(url.trim()).map_err(|_| {
VariantLockError::InvalidResolvedFormat(resolved.to_string())
})?,
),
))
} else {
Err(VariantLockError::InvalidResolvedFormat(
resolved.to_string(),
))
}
}
}
impl<'de> Deserialize<'de> for VariantLockResolved {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
Self::from_str(&s).map_err(serde::de::Error::custom)
}
}

View File

@ -0,0 +1,62 @@
use crate::variants_json::Variant;
use uv_distribution_filename::VariantLabel;
use uv_pep508::{MarkerVariantsEnvironment, VariantFeature, VariantNamespace, VariantValue};
#[derive(Debug, Default)]
pub struct VariantWithLabel {
pub variant: Variant,
pub label: Option<VariantLabel>,
}
impl MarkerVariantsEnvironment for VariantWithLabel {
fn contains_namespace(&self, namespace: &VariantNamespace) -> bool {
self.variant.contains_namespace(namespace)
}
fn contains_feature(&self, namespace: &VariantNamespace, feature: &VariantFeature) -> bool {
self.variant.contains_feature(namespace, feature)
}
fn contains_property(
&self,
namespace: &VariantNamespace,
feature: &VariantFeature,
value: &VariantValue,
) -> bool {
self.variant.contains_property(namespace, feature, value)
}
fn contains_base_namespace(&self, base: &str, namespace: &VariantNamespace) -> bool {
self.variant.contains_base_namespace(base, namespace)
}
fn contains_base_feature(
&self,
base: &str,
namespace: &VariantNamespace,
feature: &VariantFeature,
) -> bool {
self.variant.contains_base_feature(base, namespace, feature)
}
fn contains_base_property(
&self,
base: &str,
namespace: &VariantNamespace,
feature: &VariantFeature,
value: &VariantValue,
) -> bool {
self.variant
.contains_base_property(base, namespace, feature, value)
}
fn label(&self) -> Option<&str> {
self.label
.as_ref()
.map(uv_distribution_filename::VariantLabel::as_str)
}
fn is_universal(&self) -> bool {
false
}
}

View File

@ -0,0 +1,170 @@
use std::collections::BTreeMap;
use std::ops::Deref;
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};
use uv_distribution_filename::VariantLabel;
use uv_pep508::{
MarkerTree, MarkerVariantsEnvironment, Requirement, VariantFeature, VariantNamespace,
VariantValue,
};
use uv_pypi_types::VerbatimParsedUrl;
/// Mapping of namespaces in a variant
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(transparent)]
pub struct Variant(BTreeMap<VariantNamespace, BTreeMap<VariantFeature, Vec<VariantValue>>>);
impl MarkerVariantsEnvironment for Variant {
fn contains_namespace(&self, namespace: &VariantNamespace) -> bool {
self.0.contains_key(namespace)
}
fn contains_feature(&self, namespace: &VariantNamespace, feature: &VariantFeature) -> bool {
let Some(features) = self.0.get(namespace) else {
return false;
};
let Some(properties) = features.get(feature) else {
return false;
};
!properties.is_empty()
}
fn contains_property(
&self,
namespace: &VariantNamespace,
feature: &VariantFeature,
value: &VariantValue,
) -> bool {
let Some(features) = self.0.get(namespace) else {
return false;
};
let Some(values) = features.get(feature) else {
return false;
};
values.iter().any(|values| values == value)
}
fn contains_base_namespace(&self, _prefix: &str, _namespace: &VariantNamespace) -> bool {
false
}
fn contains_base_feature(
&self,
_prefix: &str,
_namespace: &VariantNamespace,
_feature: &VariantFeature,
) -> bool {
false
}
fn contains_base_property(
&self,
_prefix: &str,
_namespace: &VariantNamespace,
_feature: &VariantFeature,
_value: &VariantValue,
) -> bool {
false
}
}
impl Deref for Variant {
type Target = BTreeMap<VariantNamespace, BTreeMap<VariantFeature, Vec<VariantValue>>>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
/// Combined index metadata for wheel variants.
///
/// See <https://wheelnext.dev/variants.json>
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct VariantsJsonContent {
/// Default provider priorities.
pub default_priorities: DefaultPriorities,
/// Mapping of namespaces to provider information.
pub providers: FxHashMap<VariantNamespace, Provider>,
/// The supported, ordered properties for `AoT` providers.
pub static_properties: Option<Variant>,
/// Mapping of variant labels to properties.
pub variants: FxHashMap<VariantLabel, Variant>,
}
/// A `{name}-{version}.dist-info/variant.json` file.
#[derive(Debug, Clone, serde::Deserialize)]
#[allow(clippy::zero_sized_map_values)]
pub struct DistInfoVariantsJson {
pub variants: FxHashMap<VariantLabel, serde::de::IgnoredAny>,
}
impl DistInfoVariantsJson {
/// Returns the label for the current variant.
pub fn label(&self) -> Option<&VariantLabel> {
let mut keys = self.variants.keys();
let label = keys.next()?;
if keys.next().is_some() {
None
} else {
Some(label)
}
}
}
/// Default provider priorities
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct DefaultPriorities {
/// Default namespace priorities
pub namespace: Vec<VariantNamespace>,
/// Default feature priorities
#[serde(default)]
pub feature: BTreeMap<VariantNamespace, Vec<VariantFeature>>,
/// Default property priorities
#[serde(default)]
pub property: BTreeMap<VariantNamespace, BTreeMap<VariantFeature, Vec<VariantValue>>>,
}
/// A `namespace :: feature :: property` entry.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct VariantPropertyType {
pub namespace: VariantNamespace,
pub feature: VariantFeature,
pub value: VariantValue,
}
/// Provider information
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct Provider {
/// Environment marker specifying when to enable the plugin.
#[serde(
skip_serializing_if = "uv_pep508::marker::ser::is_empty",
serialize_with = "uv_pep508::marker::ser::serialize",
default
)]
pub enable_if: MarkerTree,
/// Whether this is an install-time provider. `false` means that it is an `AoT` provider instead.
///
/// Defaults to `true`
pub install_time: Option<bool>,
/// Whether this is an optional provider.
///
/// If it is `true`, the provider is not used unless the user opts in to it.
///
/// Defaults to `false`
#[serde(default)]
pub optional: bool,
/// Object reference to plugin class
pub plugin_api: Option<String>,
/// Dependency specifiers for how to install the plugin
pub requires: Option<Vec<Requirement<VerbatimParsedUrl>>>,
}

View File

@ -60,6 +60,7 @@ uv-tool = { workspace = true }
uv-torch = { workspace = true } uv-torch = { workspace = true }
uv-trampoline-builder = { workspace = true } uv-trampoline-builder = { workspace = true }
uv-types = { workspace = true } uv-types = { workspace = true }
uv-variants = { workspace = true }
uv-version = { workspace = true } uv-version = { workspace = true }
uv-virtualenv = { workspace = true } uv-virtualenv = { workspace = true }
uv-warnings = { workspace = true } uv-warnings = { workspace = true }

View File

@ -68,7 +68,7 @@ impl LatestClient<'_> {
// Determine whether there's a compatible wheel and/or source distribution. // Determine whether there's a compatible wheel and/or source distribution.
let mut best = None; let mut best = None;
for (filename, file) in files.all() { for (filename, file) in files.dists() {
// Skip distributions uploaded after the cutoff. // Skip distributions uploaded after the cutoff.
if let Some(exclude_newer) = self.exclude_newer.exclude_newer_package(package) { if let Some(exclude_newer) = self.exclude_newer.exclude_newer_package(package) {
match file.upload_time_utc_ms.as_ref() { match file.upload_time_utc_ms.as_ref() {

Some files were not shown because too many files have changed in this diff Show More