diff --git a/Cargo.lock b/Cargo.lock index cfd17694e..e8c17f964 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4896,6 +4896,7 @@ dependencies = [ "uv-distribution", "uv-extract", "uv-fs", + "uv-git", "uv-interpreter", "uv-normalize", "uv-types", diff --git a/crates/distribution-types/src/lib.rs b/crates/distribution-types/src/lib.rs index 313480e0d..44310d79e 100644 --- a/crates/distribution-types/src/lib.rs +++ b/crates/distribution-types/src/lib.rs @@ -41,7 +41,8 @@ use url::Url; use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename}; use pep440_rs::Version; -use pep508_rs::{Pep508Url, Scheme, VerbatimUrl}; +use pep508_rs::{Pep508Url, VerbatimUrl}; +use uv_git::GitUrl; use uv_normalize::PackageName; pub use crate::annotation::*; @@ -126,9 +127,9 @@ impl std::fmt::Display for InstalledVersion<'_> { } } -/// Either a built distribution, a wheel, or a source distribution that exists at some location +/// Either a built distribution, a wheel, or a source distribution that exists at some location. /// -/// The location can be index, url or path (wheel) or index, url, path or git (source distribution) +/// The location can be an index, URL or path (wheel), or index, URL, path or Git repository (source distribution). #[derive(Debug, Clone)] pub enum Dist { Built(BuiltDist), @@ -169,6 +170,10 @@ pub struct DirectUrlBuiltDist { /// We require that wheel urls end in the full wheel filename, e.g. /// `https://example.org/packages/flask-3.0.0-py3-none-any.whl` pub filename: WheelFilename, + /// The URL without the subdirectory fragment. + pub location: Url, + pub subdirectory: Option, + /// The URL with the subdirectory fragment. pub url: VerbatimUrl, } @@ -194,6 +199,10 @@ pub struct DirectUrlSourceDist { /// Unlike [`DirectUrlBuiltDist`], we can't require a full filename with a version here, people /// like using e.g. `foo @ https://github.com/org/repo/archive/master.zip` pub name: PackageName, + /// The URL without the subdirectory fragment. + pub location: Url, + pub subdirectory: Option, + /// The URL with the subdirectory fragment. pub url: VerbatimUrl, } @@ -201,6 +210,10 @@ pub struct DirectUrlSourceDist { #[derive(Debug, Clone)] pub struct GitSourceDist { pub name: PackageName, + /// The URL without the revision and subdirectory fragment. + pub git: Box, + pub subdirectory: Option, + /// The URL with the revision and subdirectory fragment. pub url: VerbatimUrl, } @@ -244,7 +257,12 @@ impl Dist { /// A remote built distribution (`.whl`) or source distribution from a `http://` or `https://` /// URL. - pub fn from_http_url(name: PackageName, url: VerbatimUrl) -> Result { + pub fn from_http_url( + name: PackageName, + location: Url, + subdirectory: Option, + url: VerbatimUrl, + ) -> Result { if Path::new(url.path()) .extension() .is_some_and(|ext| ext.eq_ignore_ascii_case("whl")) @@ -261,11 +279,15 @@ impl Dist { Ok(Self::Built(BuiltDist::DirectUrl(DirectUrlBuiltDist { filename, + location, + subdirectory, url, }))) } else { Ok(Self::Source(SourceDist::DirectUrl(DirectUrlSourceDist { name, + location, + subdirectory, url, }))) } @@ -274,15 +296,12 @@ impl Dist { /// A local built or source distribution from a `file://` URL. pub fn from_file_url( name: PackageName, - url: VerbatimUrl, + path: &Path, editable: bool, + url: VerbatimUrl, ) -> Result { // Store the canonicalized path, which also serves to validate that it exists. - let path = match url - .to_file_path() - .map_err(|()| Error::UrlFilename(url.to_url()))? - .canonicalize() - { + let path = match path.canonicalize() { Ok(path) => path, Err(err) if err.kind() == std::io::ErrorKind::NotFound => { return Err(Error::NotFound(url.to_url())); @@ -335,67 +354,32 @@ impl Dist { } /// A remote source distribution from a `git+https://` or `git+ssh://` url. - pub fn from_git_url(name: PackageName, url: VerbatimUrl) -> Result { - Ok(Self::Source(SourceDist::Git(GitSourceDist { name, url }))) + pub fn from_git_url( + name: PackageName, + url: VerbatimUrl, + git: GitUrl, + subdirectory: Option, + ) -> Result { + Ok(Self::Source(SourceDist::Git(GitSourceDist { + name, + git: Box::new(git), + subdirectory, + url, + }))) } - // TODO(konsti): We should carry the parsed URL through the codebase. /// Create a [`Dist`] for a URL-based distribution. pub fn from_url(name: PackageName, url: VerbatimParsedUrl) -> Result { - match Scheme::parse(url.verbatim.scheme()) { - Some(Scheme::Http | Scheme::Https) => Self::from_http_url(name, url.verbatim), - Some(Scheme::File) => Self::from_file_url(name, url.verbatim, false), - Some(Scheme::GitSsh | Scheme::GitHttps) => Self::from_git_url(name, url.verbatim), - Some(Scheme::GitGit | Scheme::GitHttp) => Err(Error::UnsupportedScheme( - url.verbatim.scheme().to_owned(), - url.verbatim.verbatim().to_string(), - "insecure Git protocol; use `git+https` or `git+ssh` instead".to_string(), - )), - Some(Scheme::GitFile) => Err(Error::UnsupportedScheme( - url.verbatim.scheme().to_owned(), - url.verbatim.verbatim().to_string(), - "local Git protocol".to_string(), - )), - Some( - Scheme::BzrHttp - | Scheme::BzrHttps - | Scheme::BzrSsh - | Scheme::BzrSftp - | Scheme::BzrFtp - | Scheme::BzrLp - | Scheme::BzrFile, - ) => Err(Error::UnsupportedScheme( - url.verbatim.scheme().to_owned(), - url.verbatim.verbatim().to_string(), - "Bazaar is not supported".to_string(), - )), - Some( - Scheme::HgFile - | Scheme::HgHttp - | Scheme::HgHttps - | Scheme::HgSsh - | Scheme::HgStaticHttp, - ) => Err(Error::UnsupportedScheme( - url.verbatim.scheme().to_owned(), - url.verbatim.verbatim().to_string(), - "Mercurial is not supported".to_string(), - )), - Some( - Scheme::SvnSsh - | Scheme::SvnHttp - | Scheme::SvnHttps - | Scheme::SvnSvn - | Scheme::SvnFile, - ) => Err(Error::UnsupportedScheme( - url.verbatim.scheme().to_owned(), - url.verbatim.verbatim().to_string(), - "Subversion is not supported".to_string(), - )), - None => Err(Error::UnsupportedScheme( - url.verbatim.scheme().to_owned(), - url.verbatim.verbatim().to_string(), - "unknown scheme".to_string(), - )), + match url.parsed_url { + ParsedUrl::Archive(archive) => { + Self::from_http_url(name, archive.url, archive.subdirectory, url.verbatim) + } + ParsedUrl::LocalFile(file) => { + Self::from_file_url(name, &file.path, false, url.verbatim) + } + ParsedUrl::Git(git) => { + Self::from_git_url(name, url.verbatim, git.url, git.subdirectory) + } } } @@ -1150,20 +1134,18 @@ mod test { /// Ensure that we don't accidentally grow the `Dist` sizes. #[test] fn dist_size() { - // At time of writing, Unix is at 240, Windows is at 248. assert!( - std::mem::size_of::() <= 248, + std::mem::size_of::() <= 336, "{}", std::mem::size_of::() ); assert!( - std::mem::size_of::() <= 248, + std::mem::size_of::() <= 336, "{}", std::mem::size_of::() ); - // At time of writing, unix is at 168, windows is at 176. assert!( - std::mem::size_of::() <= 176, + std::mem::size_of::() <= 256, "{}", std::mem::size_of::() ); diff --git a/crates/uv-client/tests/remote_metadata.rs b/crates/uv-client/tests/remote_metadata.rs index 127633334..ffa6765a1 100644 --- a/crates/uv-client/tests/remote_metadata.rs +++ b/crates/uv-client/tests/remote_metadata.rs @@ -1,6 +1,7 @@ use std::str::FromStr; use anyhow::Result; +use url::Url; use distribution_filename::WheelFilename; use distribution_types::{BuiltDist, DirectUrlBuiltDist}; @@ -20,6 +21,8 @@ async fn remote_metadata_with_and_without_cache() -> Result<()> { let filename = WheelFilename::from_str(url.rsplit_once('/').unwrap().1)?; let dist = BuiltDist::DirectUrl(DirectUrlBuiltDist { filename, + location: Url::parse(url).unwrap(), + subdirectory: None, url: VerbatimUrl::from_str(url).unwrap(), }); let metadata = client.wheel_metadata(&dist).await.unwrap(); diff --git a/crates/uv-dev/src/wheel_metadata.rs b/crates/uv-dev/src/wheel_metadata.rs index 64305cf1a..9b6788634 100644 --- a/crates/uv-dev/src/wheel_metadata.rs +++ b/crates/uv-dev/src/wheel_metadata.rs @@ -1,11 +1,11 @@ use std::str::FromStr; use anstream::println; -use anyhow::Result; +use anyhow::{bail, Result}; use clap::Parser; use distribution_filename::WheelFilename; -use distribution_types::{BuiltDist, DirectUrlBuiltDist}; +use distribution_types::{BuiltDist, DirectUrlBuiltDist, ParsedUrl}; use pep508_rs::VerbatimUrl; use uv_cache::{Cache, CacheArgs}; use uv_client::RegistryClientBuilder; @@ -30,9 +30,15 @@ pub(crate) async fn wheel_metadata(args: WheelMetadataArgs) -> Result<()> { .1, )?; + let ParsedUrl::Archive(archive) = ParsedUrl::try_from(args.url.to_url())? else { + bail!("Only HTTPS is supported"); + }; + let metadata = client .wheel_metadata(&BuiltDist::DirectUrl(DirectUrlBuiltDist { filename, + location: archive.url, + subdirectory: archive.subdirectory, url: args.url, })) .await?; diff --git a/crates/uv-installer/Cargo.toml b/crates/uv-installer/Cargo.toml index a03939ae4..b62cda5c6 100644 --- a/crates/uv-installer/Cargo.toml +++ b/crates/uv-installer/Cargo.toml @@ -27,6 +27,7 @@ uv-configuration = { workspace = true } uv-distribution = { workspace = true } uv-extract = { workspace = true } uv-fs = { workspace = true } +uv-git = { workspace = true } uv-interpreter = { workspace = true } uv-normalize = { workspace = true } uv-types = { workspace = true } diff --git a/crates/uv-installer/src/plan.rs b/crates/uv-installer/src/plan.rs index b4d13c94e..cfb1d2db5 100644 --- a/crates/uv-installer/src/plan.rs +++ b/crates/uv-installer/src/plan.rs @@ -21,6 +21,7 @@ use uv_distribution::{ BuiltWheelIndex, HttpArchivePointer, LocalArchivePointer, RegistryWheelIndex, }; use uv_fs::Simplified; +use uv_git::GitUrl; use uv_interpreter::PythonEnvironment; use uv_types::HashStrategy; @@ -224,7 +225,11 @@ impl<'a> Planner<'a> { continue; } } - RequirementSource::Url { url, .. } => { + RequirementSource::Url { + subdirectory, + location, + url, + } => { // Check if we have a wheel or a source distribution. if Path::new(url.path()) .extension() @@ -243,6 +248,8 @@ impl<'a> Planner<'a> { let wheel = DirectUrlBuiltDist { filename, + location: location.clone(), + subdirectory: subdirectory.clone(), url: url.clone(), }; @@ -288,6 +295,8 @@ impl<'a> Planner<'a> { } else { let sdist = DirectUrlSourceDist { name: requirement.name.clone(), + location: location.clone(), + subdirectory: subdirectory.clone(), url: url.clone(), }; // Find the most-compatible wheel from the cache, since we don't know @@ -300,9 +309,16 @@ impl<'a> Planner<'a> { } } } - RequirementSource::Git { url, .. } => { + RequirementSource::Git { + repository, + reference, + subdirectory, + url, + } => { let sdist = GitSourceDist { name: requirement.name.clone(), + git: Box::new(GitUrl::new(repository.clone(), reference.clone())), + subdirectory: subdirectory.clone(), url: url.clone(), }; // Find the most-compatible wheel from the cache, since we don't know diff --git a/crates/uv-requirements/src/lookahead.rs b/crates/uv-requirements/src/lookahead.rs index f9f00cb93..fe4c1a370 100644 --- a/crates/uv-requirements/src/lookahead.rs +++ b/crates/uv-requirements/src/lookahead.rs @@ -13,6 +13,7 @@ use pep508_rs::MarkerEnvironment; use pypi_types::Metadata23; use uv_configuration::{Constraints, Overrides}; use uv_distribution::{DistributionDatabase, Reporter}; +use uv_git::GitUrl; use uv_resolver::{InMemoryIndex, MetadataResponse}; use uv_types::{BuildContext, HashStrategy, RequestedRequirements}; @@ -162,16 +163,28 @@ impl<'a, Context: BuildContext> LookaheadResolver<'a, Context> { // buildable distribution. let dist = match requirement.source { RequirementSource::Registry { .. } => return Ok(None), - RequirementSource::Url { url, .. } => Dist::from_http_url(requirement.name, url)?, - RequirementSource::Git { url, .. } => Dist::Source(SourceDist::Git(GitSourceDist { + RequirementSource::Url { + subdirectory, + location, + url, + } => Dist::from_http_url(requirement.name, location, subdirectory, url)?, + RequirementSource::Git { + repository, + reference, + subdirectory, + url, + } => Dist::Source(SourceDist::Git(GitSourceDist { name: requirement.name, + git: Box::new(GitUrl::new(repository, reference)), + subdirectory, url, })), RequirementSource::Path { - path: _, + path, url, + // TODO(konsti): Figure out why we lose the editable here (does it matter?) editable: _, - } => Dist::from_file_url(requirement.name, url, false)?, + } => Dist::from_file_url(requirement.name, &path, false, url)?, }; // Fetch the metadata for the distribution. diff --git a/crates/uv-requirements/src/unnamed.rs b/crates/uv-requirements/src/unnamed.rs index f535a89de..2c22cb867 100644 --- a/crates/uv-requirements/src/unnamed.rs +++ b/crates/uv-requirements/src/unnamed.rs @@ -238,9 +238,11 @@ impl<'a, Context: BuildContext> NamedRequirementsResolver<'a, Context> { Some(Scheme::Http | Scheme::Https) => SourceUrl::Direct(DirectSourceUrl { url: &requirement.url, }), - Some(Scheme::GitSsh | Scheme::GitHttps) => SourceUrl::Git(GitSourceUrl { - url: &requirement.url, - }), + Some(Scheme::GitSsh | Scheme::GitHttps | Scheme::GitHttp) => { + SourceUrl::Git(GitSourceUrl { + url: &requirement.url, + }) + } _ => { return Err(anyhow::anyhow!( "Unsupported scheme for unnamed requirement: {}",