diff --git a/Cargo.lock b/Cargo.lock index 664f25651..242fe8462 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -406,6 +406,7 @@ dependencies = [ "uv-configuration", "uv-dispatch", "uv-distribution", + "uv-git", "uv-interpreter", "uv-resolver", "uv-types", @@ -4425,6 +4426,7 @@ dependencies = [ "uv-dispatch", "uv-distribution", "uv-fs", + "uv-git", "uv-installer", "uv-interpreter", "uv-normalize", @@ -4618,6 +4620,7 @@ dependencies = [ "uv-dispatch", "uv-distribution", "uv-fs", + "uv-git", "uv-installer", "uv-interpreter", "uv-resolver", @@ -4643,6 +4646,7 @@ dependencies = [ "uv-client", "uv-configuration", "uv-distribution", + "uv-git", "uv-installer", "uv-interpreter", "uv-resolver", @@ -4654,7 +4658,6 @@ name = "uv-distribution" version = "0.0.1" dependencies = [ "anyhow", - "cache-key", "distribution-filename", "distribution-types", "fs-err", @@ -4664,7 +4667,6 @@ dependencies = [ "insta", "install-wheel-rs", "nanoid", - "once_cell", "path-absolutize", "pep440_rs", "pep508_rs", @@ -4745,6 +4747,8 @@ dependencies = [ "anyhow", "cache-key", "cargo-util", + "dashmap", + "fs-err", "reqwest", "thiserror", "tokio", @@ -4954,6 +4958,7 @@ dependencies = [ "url", "uv-cache", "uv-configuration", + "uv-git", "uv-interpreter", "uv-normalize", ] diff --git a/crates/bench/Cargo.toml b/crates/bench/Cargo.toml index 5d49a9bbd..5c863c42b 100644 --- a/crates/bench/Cargo.toml +++ b/crates/bench/Cargo.toml @@ -39,6 +39,7 @@ uv-client = { workspace = true } uv-configuration = { workspace = true } uv-dispatch = { workspace = true } uv-distribution = { workspace = true } +uv-git = { workspace = true } uv-interpreter = { workspace = true } uv-resolver = { workspace = true } uv-types = { workspace = true } diff --git a/crates/bench/benches/uv.rs b/crates/bench/benches/uv.rs index 391007118..a7d96417d 100644 --- a/crates/bench/benches/uv.rs +++ b/crates/bench/benches/uv.rs @@ -85,6 +85,7 @@ mod resolver { }; use uv_dispatch::BuildDispatch; use uv_distribution::DistributionDatabase; + use uv_git::GitResolver; use uv_interpreter::PythonEnvironment; use uv_resolver::{ FlatIndex, InMemoryIndex, Manifest, Options, PythonRequirement, ResolutionGraph, Resolver, @@ -124,17 +125,18 @@ mod resolver { client: &RegistryClient, venv: &PythonEnvironment, ) -> Result { + let build_isolation = BuildIsolation::Isolated; + let concurrency = Concurrency::default(); + let config_settings = ConfigSettings::default(); let flat_index = FlatIndex::default(); - let index = InMemoryIndex::default(); + let git = GitResolver::default(); let hashes = HashStrategy::None; - let index_locations = IndexLocations::default(); let in_flight = InFlight::default(); + let index = InMemoryIndex::default(); + let index_locations = IndexLocations::default(); let installed_packages = EmptyInstalledPackages; let interpreter = venv.interpreter().clone(); let python_requirement = PythonRequirement::from_marker_environment(&interpreter, &MARKERS); - let concurrency = Concurrency::default(); - let config_settings = ConfigSettings::default(); - let build_isolation = BuildIsolation::Isolated; let build_context = BuildDispatch::new( client, @@ -143,6 +145,7 @@ mod resolver { &index_locations, &flat_index, &index, + &git, &in_flight, SetupPyStrategy::default(), &config_settings, diff --git a/crates/uv-dev/Cargo.toml b/crates/uv-dev/Cargo.toml index c78e8acfe..65ebb092f 100644 --- a/crates/uv-dev/Cargo.toml +++ b/crates/uv-dev/Cargo.toml @@ -25,9 +25,10 @@ uv-build = { workspace = true } uv-cache = { workspace = true, features = ["clap"] } uv-client = { workspace = true } uv-configuration = { workspace = true } -uv-distribution = { workspace = true, features = ["schemars"] } uv-dispatch = { workspace = true } +uv-distribution = { workspace = true, features = ["schemars"] } uv-fs = { workspace = true } +uv-git = { workspace = true } uv-installer = { workspace = true } uv-interpreter = { workspace = true } uv-resolver = { workspace = true } diff --git a/crates/uv-dev/src/build.rs b/crates/uv-dev/src/build.rs index 503c030cf..73cdfe61e 100644 --- a/crates/uv-dev/src/build.rs +++ b/crates/uv-dev/src/build.rs @@ -14,6 +14,7 @@ use uv_configuration::{ BuildKind, Concurrency, ConfigSettings, NoBinary, NoBuild, PreviewMode, SetupPyStrategy, }; use uv_dispatch::BuildDispatch; +use uv_git::GitResolver; use uv_interpreter::PythonEnvironment; use uv_resolver::{FlatIndex, InMemoryIndex}; use uv_types::{BuildContext, BuildIsolation, InFlight}; @@ -55,15 +56,16 @@ pub(crate) async fn build(args: BuildArgs) -> Result { let cache = Cache::try_from(args.cache_args)?.init()?; - let venv = PythonEnvironment::from_virtualenv(&cache)?; let client = RegistryClientBuilder::new(cache.clone()).build(); - let index_urls = IndexLocations::default(); - let flat_index = FlatIndex::default(); - let index = InMemoryIndex::default(); - let setup_py = SetupPyStrategy::default(); - let in_flight = InFlight::default(); - let config_settings = ConfigSettings::default(); let concurrency = Concurrency::default(); + let config_settings = ConfigSettings::default(); + let flat_index = FlatIndex::default(); + let git = GitResolver::default(); + let in_flight = InFlight::default(); + let index = InMemoryIndex::default(); + let index_urls = IndexLocations::default(); + let setup_py = SetupPyStrategy::default(); + let venv = PythonEnvironment::from_virtualenv(&cache)?; let build_dispatch = BuildDispatch::new( &client, @@ -72,6 +74,7 @@ pub(crate) async fn build(args: BuildArgs) -> Result { &index_urls, &flat_index, &index, + &git, &in_flight, setup_py, &config_settings, diff --git a/crates/uv-dispatch/Cargo.toml b/crates/uv-dispatch/Cargo.toml index 5142b695f..222052409 100644 --- a/crates/uv-dispatch/Cargo.toml +++ b/crates/uv-dispatch/Cargo.toml @@ -22,6 +22,7 @@ uv-cache = { workspace = true } uv-client = { workspace = true } uv-configuration = { workspace = true } uv-distribution = { workspace = true } +uv-git = { workspace = true } uv-installer = { workspace = true } uv-interpreter = { workspace = true } uv-resolver = { workspace = true } diff --git a/crates/uv-dispatch/src/lib.rs b/crates/uv-dispatch/src/lib.rs index 2a1e222e7..44b2bcbb9 100644 --- a/crates/uv-dispatch/src/lib.rs +++ b/crates/uv-dispatch/src/lib.rs @@ -19,6 +19,7 @@ use uv_client::RegistryClient; use uv_configuration::{BuildKind, ConfigSettings, NoBinary, NoBuild, Reinstall, SetupPyStrategy}; use uv_configuration::{Concurrency, PreviewMode}; use uv_distribution::DistributionDatabase; +use uv_git::GitResolver; use uv_installer::{Downloader, Installer, Plan, Planner, SitePackages}; use uv_interpreter::{Interpreter, PythonEnvironment}; use uv_resolver::{FlatIndex, InMemoryIndex, Manifest, Options, PythonRequirement, Resolver}; @@ -33,6 +34,7 @@ pub struct BuildDispatch<'a> { index_locations: &'a IndexLocations, flat_index: &'a FlatIndex, index: &'a InMemoryIndex, + git: &'a GitResolver, in_flight: &'a InFlight, setup_py: SetupPyStrategy, build_isolation: BuildIsolation<'a>, @@ -56,6 +58,7 @@ impl<'a> BuildDispatch<'a> { index_locations: &'a IndexLocations, flat_index: &'a FlatIndex, index: &'a InMemoryIndex, + git: &'a GitResolver, in_flight: &'a InFlight, setup_py: SetupPyStrategy, config_settings: &'a ConfigSettings, @@ -73,6 +76,7 @@ impl<'a> BuildDispatch<'a> { index_locations, flat_index, index, + git, in_flight, setup_py, config_settings, @@ -102,6 +106,10 @@ impl<'a> BuildContext for BuildDispatch<'a> { self.cache } + fn git(&self) -> &GitResolver { + self.git + } + fn interpreter(&self) -> &Interpreter { self.interpreter } @@ -194,9 +202,9 @@ impl<'a> BuildContext for BuildDispatch<'a> { extraneous: _, } = Planner::new(&requirements).build( site_packages, - &Reinstall::None, - &NoBinary::None, - &HashStrategy::None, + &Reinstall::default(), + &NoBinary::default(), + &HashStrategy::default(), self.index_locations, self.cache(), venv, diff --git a/crates/uv-distribution/Cargo.toml b/crates/uv-distribution/Cargo.toml index 0adc6eda2..580b54bc8 100644 --- a/crates/uv-distribution/Cargo.toml +++ b/crates/uv-distribution/Cargo.toml @@ -13,7 +13,6 @@ license = { workspace = true } workspace = true [dependencies] -cache-key = { workspace = true } distribution-filename = { workspace = true } distribution-types = { workspace = true } install-wheel-rs = { workspace = true } @@ -36,7 +35,6 @@ fs-err = { workspace = true } futures = { workspace = true } glob = { workspace = true } nanoid = { workspace = true } -once_cell = { workspace = true } path-absolutize = { workspace = true } reqwest = { workspace = true } reqwest-middleware = { workspace = true } diff --git a/crates/uv-distribution/src/error.rs b/crates/uv-distribution/src/error.rs index 14a02b823..81452a37e 100644 --- a/crates/uv-distribution/src/error.rs +++ b/crates/uv-distribution/src/error.rs @@ -25,8 +25,8 @@ pub enum Error { RelativePath(PathBuf), #[error(transparent)] JoinRelativeUrl(#[from] pypi_types::JoinRelativeError), - #[error("Git operation failed")] - Git(#[source] anyhow::Error), + #[error(transparent)] + Git(#[from] uv_git::GitResolverError), #[error(transparent)] Reqwest(#[from] BetterReqwestError), #[error(transparent)] diff --git a/crates/uv-distribution/src/git.rs b/crates/uv-distribution/src/git.rs deleted file mode 100644 index a35bfeb2c..000000000 --- a/crates/uv-distribution/src/git.rs +++ /dev/null @@ -1,288 +0,0 @@ -use std::sync::{Arc, Mutex}; - -use anyhow::Result; -use fs_err::tokio as fs; -use once_cell::sync::Lazy; -use rustc_hash::FxHashMap; -use tracing::debug; -use url::Url; - -use cache_key::{CanonicalUrl, RepositoryUrl}; -use pypi_types::ParsedGitUrl; -use uv_cache::{Cache, CacheBucket}; -use uv_fs::LockedFile; -use uv_git::{Fetch, GitReference, GitSha, GitSource, GitUrl}; - -use crate::error::Error; -use crate::reporter::Facade; -use crate::Reporter; - -/// Global cache of resolved Git references. -/// -/// Used to ensure that a given Git URL is only resolved once, and that the resolved URL is -/// consistent across all invocations. (For example: if a Git URL refers to a branch, like `main`, -/// then the resolved URL should always refer to the same commit across the lifetime of the -/// process.) -static RESOLVED_GIT_REFS: Lazy>> = - Lazy::new(Mutex::default); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct RepositoryReference { - /// The URL of the Git repository, with any query parameters and fragments removed. - url: RepositoryUrl, - /// The reference to the commit to use, which could be a branch, tag or revision. - reference: GitReference, -} - -impl RepositoryReference { - fn new(git: &GitUrl) -> Self { - Self { - url: RepositoryUrl::new(git.repository()), - reference: git.reference().clone(), - } - } -} - -/// Download a source distribution from a Git repository. -/// -/// Assumes that the URL is a precise Git URL, with a full commit hash. -pub(crate) async fn fetch_git_archive( - url: &GitUrl, - cache: &Cache, - reporter: Option<&Arc>, -) -> Result { - debug!("Fetching source distribution from Git: {url}"); - let git_dir = cache.bucket(CacheBucket::Git); - - // Avoid races between different processes, too. - let lock_dir = git_dir.join("locks"); - fs::create_dir_all(&lock_dir) - .await - .map_err(Error::CacheWrite)?; - let repository_url = RepositoryUrl::new(url.repository()); - let _lock = LockedFile::acquire( - lock_dir.join(cache_key::digest(&repository_url)), - &repository_url, - ) - .map_err(Error::CacheWrite)?; - - // Fetch the Git repository. - let source = if let Some(reporter) = reporter { - GitSource::new(url.clone(), git_dir).with_reporter(Facade::from(reporter.clone())) - } else { - GitSource::new(url.clone(), git_dir) - }; - let fetch = tokio::task::spawn_blocking(move || source.fetch()) - .await? - .map_err(Error::Git)?; - - Ok(fetch) -} - -/// Given a remote source distribution, return a precise variant, if possible. -/// -/// For example, given a Git dependency with a reference to a branch or tag, return a URL -/// with a precise reference to the current commit of that branch or tag. -/// -/// This method takes into account various normalizations that are independent from the Git -/// layer. For example: removing `#subdirectory=pkg_dir`-like fragments, and removing `git+` -/// prefix kinds. -pub(crate) async fn resolve_precise( - url: &GitUrl, - cache: &Cache, - reporter: Option<&Arc>, -) -> Result, Error> { - // If the Git reference already contains a complete SHA, short-circuit. - if url.precise().is_some() { - return Ok(None); - } - - // If the Git reference is in the in-memory cache, return it. - { - let resolved_git_refs = RESOLVED_GIT_REFS.lock().unwrap(); - let reference = RepositoryReference::new(url); - if let Some(precise) = resolved_git_refs.get(&reference) { - return Ok(Some(url.clone().with_precise(*precise))); - } - } - - let git_dir = cache.bucket(CacheBucket::Git); - - // Fetch the precise SHA of the Git reference (which could be a branch, a tag, a partial - // commit, etc.). - let source = if let Some(reporter) = reporter { - GitSource::new(url.clone(), git_dir).with_reporter(Facade::from(reporter.clone())) - } else { - GitSource::new(url.clone(), git_dir) - }; - let fetch = tokio::task::spawn_blocking(move || source.fetch()) - .await? - .map_err(Error::Git)?; - let git = fetch.into_git(); - - // Insert the resolved URL into the in-memory cache. - if let Some(precise) = git.precise() { - let mut resolved_git_refs = RESOLVED_GIT_REFS.lock().unwrap(); - let reference = RepositoryReference::new(url); - resolved_git_refs.insert(reference, precise); - } - - Ok(Some(git)) -} - -/// Given a remote source distribution, return a precise variant, if possible. -/// -/// For example, given a Git dependency with a reference to a branch or tag, return a URL -/// with a precise reference to the current commit of that branch or tag. -/// -/// This method takes into account various normalizations that are independent from the Git -/// layer. For example: removing `#subdirectory=pkg_dir`-like fragments, and removing `git+` -/// prefix kinds. -/// -/// This method will only return precise URLs for URLs that have already been resolved via -/// [`resolve_precise`]. -pub fn git_url_to_precise(url: GitUrl) -> Option { - let resolved_git_refs = RESOLVED_GIT_REFS.lock().unwrap(); - let reference = RepositoryReference::new(&url); - let precise = resolved_git_refs.get(&reference)?; - Some(url.with_precise(*precise)) -} - -/// Returns `true` if the URLs refer to the same Git commit. -/// -/// For example, the previous URL could be a branch or tag, while the current URL would be a -/// precise commit hash. -pub fn is_same_reference<'a>(a: &'a Url, b: &'a Url) -> bool { - let resolved_git_refs = RESOLVED_GIT_REFS.lock().unwrap(); - is_same_reference_impl(a, b, &resolved_git_refs) -} - -/// Returns `true` if the URLs refer to the same Git commit. -/// -/// Like [`is_same_reference`], but accepts a resolved reference cache for testing. -fn is_same_reference_impl<'a>( - a: &'a Url, - b: &'a Url, - resolved_refs: &FxHashMap, -) -> bool { - // Convert `a` to a Git URL, if possible. - let Ok(a_git) = ParsedGitUrl::try_from(Url::from(CanonicalUrl::new(a))) else { - return false; - }; - - // Convert `b` to a Git URL, if possible. - let Ok(b_git) = ParsedGitUrl::try_from(Url::from(CanonicalUrl::new(b))) else { - return false; - }; - - // The URLs must refer to the same subdirectory, if any. - if a_git.subdirectory != b_git.subdirectory { - return false; - } - - // Convert `a` to a repository URL. - let a_ref = RepositoryReference::new(&a_git.url); - - // Convert `b` to a repository URL. - let b_ref = RepositoryReference::new(&b_git.url); - - // The URLs must refer to the same repository. - if a_ref.url != b_ref.url { - return false; - } - - // If the URLs have the same tag, they refer to the same commit. - if a_ref.reference == b_ref.reference { - return true; - } - - // Otherwise, the URLs must resolve to the same precise commit. - let Some(a_precise) = a_git - .url - .precise() - .or_else(|| resolved_refs.get(&a_ref).copied()) - else { - return false; - }; - - let Some(b_precise) = b_git - .url - .precise() - .or_else(|| resolved_refs.get(&b_ref).copied()) - else { - return false; - }; - - a_precise == b_precise -} - -#[cfg(test)] -mod tests { - use anyhow::Result; - use rustc_hash::FxHashMap; - use std::str::FromStr; - use url::Url; - - use crate::git::RepositoryReference; - use uv_git::{GitSha, GitUrl}; - - #[test] - fn same_reference() -> Result<()> { - let empty = FxHashMap::default(); - - // Same repository, same tag. - let a = Url::parse("git+https://example.com/MyProject.git@main")?; - let b = Url::parse("git+https://example.com/MyProject.git@main")?; - assert!(super::is_same_reference_impl(&a, &b, &empty)); - - // Same repository, same tag, same subdirectory. - let a = Url::parse("git+https://example.com/MyProject.git@main#subdirectory=pkg_dir")?; - let b = Url::parse("git+https://example.com/MyProject.git@main#subdirectory=pkg_dir")?; - assert!(super::is_same_reference_impl(&a, &b, &empty)); - - // Different repositories, same tag. - let a = Url::parse("git+https://example.com/MyProject.git@main")?; - let b = Url::parse("git+https://example.com/MyOtherProject.git@main")?; - assert!(!super::is_same_reference_impl(&a, &b, &empty)); - - // Same repository, different tags. - let a = Url::parse("git+https://example.com/MyProject.git@main")?; - let b = Url::parse("git+https://example.com/MyProject.git@v1.0")?; - assert!(!super::is_same_reference_impl(&a, &b, &empty)); - - // Same repository, same tag, different subdirectory. - let a = Url::parse("git+https://example.com/MyProject.git@main#subdirectory=pkg_dir")?; - let b = Url::parse("git+https://example.com/MyProject.git@main#subdirectory=other_dir")?; - assert!(!super::is_same_reference_impl(&a, &b, &empty)); - - // Same repository, different tags, but same precise commit. - let a = Url::parse("git+https://example.com/MyProject.git@main")?; - let b = Url::parse( - "git+https://example.com/MyProject.git@164a8735b081663fede48c5041667b194da15d25", - )?; - let mut resolved_refs = FxHashMap::default(); - resolved_refs.insert( - RepositoryReference::new(&GitUrl::try_from(Url::parse( - "https://example.com/MyProject@main", - )?)?), - GitSha::from_str("164a8735b081663fede48c5041667b194da15d25")?, - ); - assert!(super::is_same_reference_impl(&a, &b, &resolved_refs)); - - // Same repository, different tags, different precise commit. - let a = Url::parse("git+https://example.com/MyProject.git@main")?; - let b = Url::parse( - "git+https://example.com/MyProject.git@164a8735b081663fede48c5041667b194da15d25", - )?; - let mut resolved_refs = FxHashMap::default(); - resolved_refs.insert( - RepositoryReference::new(&GitUrl::try_from(Url::parse( - "https://example.com/MyProject@main", - )?)?), - GitSha::from_str("f2c9e88f3ec9526bbcec68d150b176d96a750aba")?, - ); - assert!(!super::is_same_reference_impl(&a, &b, &resolved_refs)); - - Ok(()) - } -} diff --git a/crates/uv-distribution/src/lib.rs b/crates/uv-distribution/src/lib.rs index 04a97a9e1..3b0e08ea7 100644 --- a/crates/uv-distribution/src/lib.rs +++ b/crates/uv-distribution/src/lib.rs @@ -1,7 +1,6 @@ pub use distribution_database::{DistributionDatabase, HttpArchivePointer, LocalArchivePointer}; pub use download::LocalWheel; pub use error::Error; -pub use git::{git_url_to_precise, is_same_reference}; pub use index::{BuiltWheelIndex, RegistryWheelIndex}; pub use metadata::{ArchiveMetadata, Metadata}; pub use reporter::Reporter; @@ -12,7 +11,6 @@ mod archive; mod distribution_database; mod download; mod error; -mod git; mod index; mod locks; mod metadata; diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index 595966e71..e95eee3e8 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -36,8 +36,8 @@ use uv_types::{BuildContext, SourceBuildTrait}; use crate::distribution_database::ManagedClient; use crate::error::Error; -use crate::git::{fetch_git_archive, resolve_precise}; use crate::metadata::{ArchiveMetadata, Metadata}; +use crate::reporter::Facade; use crate::source::built_wheel_metadata::BuiltWheelMetadata; use crate::source::revision::Revision; use crate::Reporter; @@ -1040,12 +1040,15 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { } // Resolve to a precise Git SHA. - let url = if let Some(url) = resolve_precise( - resource.git, - self.build_context.cache(), - self.reporter.as_ref(), - ) - .await? + let url = if let Some(url) = self + .build_context + .git() + .resolve( + resource.git, + self.build_context.cache().bucket(CacheBucket::Git), + self.reporter.clone().map(Facade::from), + ) + .await? { Cow::Owned(url) } else { @@ -1053,8 +1056,15 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { }; // Fetch the Git repository. - let fetch = - fetch_git_archive(&url, self.build_context.cache(), self.reporter.as_ref()).await?; + let fetch = self + .build_context + .git() + .fetch( + &url, + self.build_context.cache().bucket(CacheBucket::Git), + self.reporter.clone().map(Facade::from), + ) + .await?; let git_sha = fetch.git().precise().expect("Exact commit after checkout"); let cache_shard = self.build_context.cache().shard( @@ -1114,12 +1124,15 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { } // Resolve to a precise Git SHA. - let url = if let Some(url) = resolve_precise( - resource.git, - self.build_context.cache(), - self.reporter.as_ref(), - ) - .await? + let url = if let Some(url) = self + .build_context + .git() + .resolve( + resource.git, + self.build_context.cache().bucket(CacheBucket::Git), + self.reporter.clone().map(Facade::from), + ) + .await? { Cow::Owned(url) } else { @@ -1127,8 +1140,15 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { }; // Fetch the Git repository. - let fetch = - fetch_git_archive(&url, self.build_context.cache(), self.reporter.as_ref()).await?; + let fetch = self + .build_context + .git() + .fetch( + &url, + self.build_context.cache().bucket(CacheBucket::Git), + self.reporter.clone().map(Facade::from), + ) + .await?; let git_sha = fetch.git().precise().expect("Exact commit after checkout"); let cache_shard = self.build_context.cache().shard( diff --git a/crates/uv-git/Cargo.toml b/crates/uv-git/Cargo.toml index b97926e37..999a6a05c 100644 --- a/crates/uv-git/Cargo.toml +++ b/crates/uv-git/Cargo.toml @@ -17,9 +17,11 @@ cache-key = { workspace = true } uv-fs = { workspace = true } anyhow = { workspace = true } -thiserror = { workspace = true } cargo-util = { workspace = true } +dashmap = { workspace = true } +fs-err = { workspace = true, features = ["tokio"] } reqwest = { workspace = true, features = ["blocking"] } +thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } url = { workspace = true } diff --git a/crates/uv-git/src/lib.rs b/crates/uv-git/src/lib.rs index a650f8a5e..dd3442501 100644 --- a/crates/uv-git/src/lib.rs +++ b/crates/uv-git/src/lib.rs @@ -2,10 +2,12 @@ use std::str::FromStr; use url::Url; pub use crate::git::GitReference; +pub use crate::resolver::{GitResolver, GitResolverError, RepositoryReference}; pub use crate::sha::{GitOid, GitSha, OidParseError}; pub use crate::source::{Fetch, GitSource, Reporter}; mod git; +mod resolver; mod sha; mod source; diff --git a/crates/uv-git/src/resolver.rs b/crates/uv-git/src/resolver.rs new file mode 100644 index 000000000..5a3689be8 --- /dev/null +++ b/crates/uv-git/src/resolver.rs @@ -0,0 +1,154 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::Result; +use dashmap::mapref::one::Ref; +use dashmap::DashMap; +use fs_err::tokio as fs; +use tracing::debug; + +use cache_key::RepositoryUrl; +use uv_fs::LockedFile; + +use crate::{Fetch, GitReference, GitSha, GitSource, GitUrl, Reporter}; + +#[derive(Debug, thiserror::Error)] +pub enum GitResolverError { + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Join(#[from] tokio::task::JoinError), + #[error("Git operation failed")] + Git(#[source] anyhow::Error), +} + +/// A resolver for Git repositories. +#[derive(Default, Clone)] +pub struct GitResolver(Arc>); + +impl GitResolver { + /// Returns the [`GitSha`] for the given [`RepositoryReference`], if it exists. + pub fn get(&self, reference: &RepositoryReference) -> Option> { + self.0.get(reference) + } + + /// Inserts a new [`GitSha`] for the given [`RepositoryReference`]. + pub fn insert(&self, reference: RepositoryReference, sha: GitSha) { + self.0.insert(reference, sha); + } + + /// Download a source distribution from a Git repository. + /// + /// Assumes that the URL is a precise Git URL, with a full commit hash. + pub async fn fetch( + &self, + url: &GitUrl, + cache: PathBuf, + reporter: Option, + ) -> Result { + debug!("Fetching source distribution from Git: {url}"); + + // Avoid races between different processes, too. + let lock_dir = cache.join("locks"); + fs::create_dir_all(&lock_dir).await?; + let repository_url = RepositoryUrl::new(url.repository()); + let _lock = LockedFile::acquire( + lock_dir.join(cache_key::digest(&repository_url)), + &repository_url, + )?; + + // Fetch the Git repository. + let source = if let Some(reporter) = reporter { + GitSource::new(url.clone(), cache).with_reporter(reporter) + } else { + GitSource::new(url.clone(), cache) + }; + let fetch = tokio::task::spawn_blocking(move || source.fetch()) + .await? + .map_err(GitResolverError::Git)?; + + Ok(fetch) + } + + /// Given a remote source distribution, return a precise variant, if possible. + /// + /// For example, given a Git dependency with a reference to a branch or tag, return a URL + /// with a precise reference to the current commit of that branch or tag. + /// + /// This method takes into account various normalizations that are independent from the Git + /// layer. For example: removing `#subdirectory=pkg_dir`-like fragments, and removing `git+` + /// prefix kinds. + pub async fn resolve( + &self, + url: &GitUrl, + cache: impl Into, + reporter: Option, + ) -> Result, GitResolverError> { + // If the Git reference already contains a complete SHA, short-circuit. + if url.precise().is_some() { + return Ok(None); + } + + // If the Git reference is in the in-memory cache, return it. + { + let reference = RepositoryReference::from(url); + if let Some(precise) = self.get(&reference) { + return Ok(Some(url.clone().with_precise(*precise))); + } + } + + // Fetch the precise SHA of the Git reference (which could be a branch, a tag, a partial + // commit, etc.). + let source = if let Some(reporter) = reporter { + GitSource::new(url.clone(), cache).with_reporter(reporter) + } else { + GitSource::new(url.clone(), cache) + }; + let fetch = tokio::task::spawn_blocking(move || source.fetch()) + .await? + .map_err(GitResolverError::Git)?; + let git = fetch.into_git(); + + // Insert the resolved URL into the in-memory cache. + if let Some(precise) = git.precise() { + let reference = RepositoryReference::from(url); + self.insert(reference, precise); + } + + Ok(Some(git)) + } + + /// Given a remote source distribution, return a precise variant, if possible. + /// + /// For example, given a Git dependency with a reference to a branch or tag, return a URL + /// with a precise reference to the current commit of that branch or tag. + /// + /// This method takes into account various normalizations that are independent from the Git + /// layer. For example: removing `#subdirectory=pkg_dir`-like fragments, and removing `git+` + /// prefix kinds. + /// + /// This method will only return precise URLs for URLs that have already been resolved via + /// [`resolve_precise`]. + pub fn precise(&self, url: GitUrl) -> Option { + let reference = RepositoryReference::from(&url); + let precise = self.get(&reference)?; + Some(url.with_precise(*precise)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct RepositoryReference { + /// The URL of the Git repository, with any query parameters and fragments removed. + pub url: RepositoryUrl, + /// The reference to the commit to use, which could be a branch, tag or revision. + pub reference: GitReference, +} + +impl From<&GitUrl> for RepositoryReference { + fn from(git: &GitUrl) -> Self { + Self { + url: RepositoryUrl::new(git.repository()), + reference: git.reference().clone(), + } + } +} diff --git a/crates/uv-resolver/src/pubgrub/dependencies.rs b/crates/uv-resolver/src/pubgrub/dependencies.rs index 8c4b6f2e6..b2343e40a 100644 --- a/crates/uv-resolver/src/pubgrub/dependencies.rs +++ b/crates/uv-resolver/src/pubgrub/dependencies.rs @@ -8,6 +8,7 @@ use pep440_rs::Version; use pep508_rs::MarkerEnvironment; use pypi_types::{Requirement, RequirementSource}; use uv_configuration::{Constraints, Overrides}; +use uv_git::GitResolver; use uv_normalize::{ExtraName, PackageName}; use crate::pubgrub::specifier::PubGrubSpecifier; @@ -29,6 +30,7 @@ impl PubGrubDependencies { source_extra: Option<&ExtraName>, urls: &Urls, locals: &Locals, + git: &GitResolver, env: Option<&MarkerEnvironment>, ) -> Result { let mut dependencies = Vec::default(); @@ -42,6 +44,7 @@ impl PubGrubDependencies { source_extra, urls, locals, + git, env, &mut dependencies, &mut seen, @@ -71,6 +74,7 @@ fn add_requirements( source_extra: Option<&ExtraName>, urls: &Urls, locals: &Locals, + git: &GitResolver, env: Option<&MarkerEnvironment>, dependencies: &mut Vec<(PubGrubPackage, Range)>, seen: &mut FxHashSet, @@ -97,9 +101,10 @@ fn add_requirements( None, urls, locals, + git, )) .chain(requirement.extras.clone().into_iter().map(|extra| { - PubGrubRequirement::from_requirement(requirement, Some(extra), urls, locals) + PubGrubRequirement::from_requirement(requirement, Some(extra), urls, locals, git) })) { let PubGrubRequirement { package, version } = result?; @@ -126,6 +131,7 @@ fn add_requirements( Some(extra), urls, locals, + git, env, dependencies, seen, @@ -156,7 +162,7 @@ fn add_requirements( // Add the package. let PubGrubRequirement { package, version } = - PubGrubRequirement::from_constraint(constraint, urls, locals)?; + PubGrubRequirement::from_constraint(constraint, urls, locals, git)?; // Ignore self-dependencies. if let PubGrubPackageInner::Package { name, .. } = &*package { @@ -196,6 +202,7 @@ impl PubGrubRequirement { extra: Option, urls: &Urls, locals: &Locals, + git: &GitResolver, ) -> Result { match &requirement.source { RequirementSource::Registry { specifier, .. } => { @@ -241,7 +248,7 @@ impl PubGrubRequirement { )); }; - if !Urls::is_allowed(&expected.verbatim, url) { + if !Urls::is_allowed(&expected.verbatim, url, git) { return Err(ResolveError::ConflictingUrlsTransitive( requirement.name.clone(), expected.verbatim.verbatim().to_string(), @@ -267,7 +274,7 @@ impl PubGrubRequirement { )); }; - if !Urls::is_allowed(&expected.verbatim, url) { + if !Urls::is_allowed(&expected.verbatim, url, git) { return Err(ResolveError::ConflictingUrlsTransitive( requirement.name.clone(), expected.verbatim.verbatim().to_string(), @@ -293,7 +300,7 @@ impl PubGrubRequirement { )); }; - if !Urls::is_allowed(&expected.verbatim, url) { + if !Urls::is_allowed(&expected.verbatim, url, git) { return Err(ResolveError::ConflictingUrlsTransitive( requirement.name.clone(), expected.verbatim.verbatim().to_string(), @@ -319,7 +326,8 @@ impl PubGrubRequirement { constraint: &Requirement, urls: &Urls, locals: &Locals, + git: &GitResolver, ) -> Result { - Self::from_requirement(constraint, None, urls, locals) + Self::from_requirement(constraint, None, urls, locals, git) } } diff --git a/crates/uv-resolver/src/redirect.rs b/crates/uv-resolver/src/redirect.rs index 714ec912b..384cb0cae 100644 --- a/crates/uv-resolver/src/redirect.rs +++ b/crates/uv-resolver/src/redirect.rs @@ -2,8 +2,39 @@ use url::Url; use pep508_rs::VerbatimUrl; use pypi_types::{ParsedGitUrl, ParsedUrl, VerbatimParsedUrl}; -use uv_distribution::git_url_to_precise; -use uv_git::GitReference; +use uv_git::{GitReference, GitResolver}; + +/// Map a URL to a precise URL, if possible. +pub(crate) fn url_to_precise(url: VerbatimParsedUrl, git: &GitResolver) -> VerbatimParsedUrl { + let ParsedUrl::Git(ParsedGitUrl { + url: git_url, + subdirectory, + }) = &url.parsed_url + else { + return url; + }; + + let Some(new_git_url) = git.precise(git_url.clone()) else { + debug_assert!( + matches!(git_url.reference(), GitReference::FullCommit(_)), + "Unseen Git URL: {}, {:?}", + url.verbatim, + git_url + ); + return url; + }; + + let new_parsed_url = ParsedGitUrl { + url: new_git_url, + subdirectory: subdirectory.clone(), + }; + let new_url = Url::from(new_parsed_url.clone()); + let new_verbatim_url = apply_redirect(&url.verbatim, new_url); + VerbatimParsedUrl { + parsed_url: ParsedUrl::Git(new_parsed_url), + verbatim: new_verbatim_url, + } +} /// Given a [`VerbatimUrl`] and a redirect, apply the redirect to the URL while preserving as much /// of the verbatim representation as possible. @@ -39,37 +70,6 @@ fn apply_redirect(url: &VerbatimUrl, redirect: Url) -> VerbatimUrl { redirect } -pub(crate) fn url_to_precise(url: VerbatimParsedUrl) -> VerbatimParsedUrl { - let ParsedUrl::Git(ParsedGitUrl { - url: git_url, - subdirectory, - }) = url.parsed_url.clone() - else { - return url; - }; - - let Some(new_git_url) = git_url_to_precise(git_url.clone()) else { - debug_assert!( - matches!(git_url.reference(), GitReference::FullCommit(_)), - "Unseen Git URL: {}, {:?}", - url.verbatim, - git_url - ); - return url; - }; - - let new_parsed_url = ParsedGitUrl { - url: new_git_url, - subdirectory, - }; - let new_url = Url::from(new_parsed_url.clone()); - let new_verbatim_url = apply_redirect(&url.verbatim, new_url); - VerbatimParsedUrl { - parsed_url: ParsedUrl::Git(new_parsed_url), - verbatim: new_verbatim_url, - } -} - #[cfg(test)] mod tests { use url::Url; diff --git a/crates/uv-resolver/src/resolution/graph.rs b/crates/uv-resolver/src/resolution/graph.rs index e8653cf54..3188e8dba 100644 --- a/crates/uv-resolver/src/resolution/graph.rs +++ b/crates/uv-resolver/src/resolution/graph.rs @@ -12,6 +12,7 @@ use distribution_types::{ use pep440_rs::{Version, VersionSpecifier}; use pep508_rs::{MarkerEnvironment, MarkerTree}; use pypi_types::{ParsedUrlError, Requirement, Yanked}; +use uv_git::GitResolver; use uv_normalize::{ExtraName, PackageName}; use crate::preferences::Preferences; @@ -40,6 +41,7 @@ impl ResolutionGraph { pub(crate) fn from_state( index: &InMemoryIndex, preferences: &Preferences, + git: &GitResolver, resolution: Resolution, ) -> anyhow::Result { // Collect all marker expressions from relevant pubgrub packages. @@ -183,7 +185,7 @@ impl ResolutionGraph { url: Some(url), } => { // Create the distribution. - let dist = Dist::from_url(name.clone(), url_to_precise(url.clone()))?; + let dist = Dist::from_url(name.clone(), url_to_precise(url.clone(), git))?; // Extract the hashes, preserving those that were already present in the // lockfile if necessary. diff --git a/crates/uv-resolver/src/resolver/mod.rs b/crates/uv-resolver/src/resolver/mod.rs index 63e5a2561..cf2132c5b 100644 --- a/crates/uv-resolver/src/resolver/mod.rs +++ b/crates/uv-resolver/src/resolver/mod.rs @@ -31,6 +31,7 @@ use pypi_types::{Metadata23, Requirement}; pub(crate) use urls::Urls; use uv_configuration::{Constraints, Overrides}; use uv_distribution::{ArchiveMetadata, DistributionDatabase}; +use uv_git::GitResolver; use uv_normalize::{ExtraName, PackageName}; use uv_types::{BuildContext, HashStrategy, InstalledPackagesProvider}; @@ -82,6 +83,7 @@ struct ResolverState { constraints: Constraints, overrides: Overrides, preferences: Preferences, + git: GitResolver, exclusions: Exclusions, urls: Urls, locals: Locals, @@ -154,6 +156,7 @@ impl<'a, Context: BuildContext, InstalledPackages: InstalledPackagesProvider> markers, python_requirement, index, + build_context.git(), provider, installed_packages, ) @@ -172,16 +175,18 @@ impl markers: Option<&MarkerEnvironment>, python_requirement: &PythonRequirement, index: &InMemoryIndex, + git: &GitResolver, provider: Provider, installed_packages: InstalledPackages, ) -> Result { let state = ResolverState { index: index.clone(), + git: git.clone(), unavailable_packages: DashMap::default(), incomplete_packages: DashMap::default(), selector: CandidateSelector::for_resolution(options, &manifest, markers), dependency_mode: options.dependency_mode, - urls: Urls::from_manifest(&manifest, markers, options.dependency_mode)?, + urls: Urls::from_manifest(&manifest, markers, git, options.dependency_mode)?, locals: Locals::from_manifest(&manifest, markers, options.dependency_mode), project: manifest.project, requirements: manifest.requirements, @@ -549,7 +554,7 @@ impl ResolverState ResolverState ResolverState, + git: &GitResolver, dependencies: DependencyMode, ) -> Result { let mut urls: FxHashMap = FxHashMap::default(); @@ -93,7 +95,7 @@ impl Urls { }; if let Some(previous) = urls.insert(requirement.name.clone(), url.clone()) { if !is_equal(&previous.verbatim, &url.verbatim) { - if is_same_reference(&previous.verbatim, &url.verbatim) { + if is_same_reference(&previous.verbatim, &url.verbatim, git) { debug!( "Allowing {} as a variant of {}", &url.verbatim, previous.verbatim @@ -120,12 +122,16 @@ impl Urls { } /// Returns `true` if the provided URL is compatible with the given "allowed" URL. - pub(crate) fn is_allowed(expected: &VerbatimUrl, provided: &VerbatimUrl) -> bool { + pub(crate) fn is_allowed( + expected: &VerbatimUrl, + provided: &VerbatimUrl, + git: &GitResolver, + ) -> bool { #[allow(clippy::if_same_then_else)] if is_equal(expected, provided) { // If the URLs are canonically equivalent, they're compatible. true - } else if is_same_reference(expected, provided) { + } else if is_same_reference(expected, provided, git) { // If the URLs refer to the same commit, they're compatible. true } else { @@ -139,12 +145,75 @@ impl Urls { /// /// Accepts URLs that map to the same [`CanonicalUrl`]. fn is_equal(previous: &VerbatimUrl, url: &VerbatimUrl) -> bool { - cache_key::CanonicalUrl::new(previous.raw()) == cache_key::CanonicalUrl::new(url.raw()) + CanonicalUrl::new(previous.raw()) == CanonicalUrl::new(url.raw()) +} + +/// Returns `true` if the URLs refer to the same Git commit. +/// +/// For example, the previous URL could be a branch or tag, while the current URL would be a +/// precise commit hash. +fn is_same_reference<'a>(a: &'a Url, b: &'a Url, git: &'a GitResolver) -> bool { + // Convert `a` to a Git URL, if possible. + let Ok(a_git) = ParsedGitUrl::try_from(Url::from(CanonicalUrl::new(a))) else { + return false; + }; + + // Convert `b` to a Git URL, if possible. + let Ok(b_git) = ParsedGitUrl::try_from(Url::from(CanonicalUrl::new(b))) else { + return false; + }; + + // The URLs must refer to the same subdirectory, if any. + if a_git.subdirectory != b_git.subdirectory { + return false; + } + + // Convert `a` to a repository URL. + let a_ref = RepositoryReference::from(&a_git.url); + + // Convert `b` to a repository URL. + let b_ref = RepositoryReference::from(&b_git.url); + + // The URLs must refer to the same repository. + if a_ref.url != b_ref.url { + return false; + } + + // If the URLs have the same tag, they refer to the same commit. + if a_ref.reference == b_ref.reference { + return true; + } + + // Otherwise, the URLs must resolve to the same precise commit. + let Some(a_precise) = a_git + .url + .precise() + .or_else(|| git.get(&a_ref).map(|sha| *sha)) + else { + return false; + }; + + let Some(b_precise) = b_git + .url + .precise() + .or_else(|| git.get(&b_ref).map(|sha| *sha)) + else { + return false; + }; + + a_precise == b_precise } #[cfg(test)] mod tests { - use super::*; + use std::str::FromStr; + + use url::Url; + + use pep508_rs::VerbatimUrl; + use uv_git::{GitResolver, GitSha, GitUrl, RepositoryReference}; + + use crate::resolver::urls::{is_equal, is_same_reference}; #[test] fn url_compatibility() -> Result<(), url::ParseError> { @@ -175,4 +244,64 @@ mod tests { Ok(()) } + + #[test] + fn same_reference() -> anyhow::Result<()> { + let empty = GitResolver::default(); + + // Same repository, same tag. + let a = Url::parse("git+https://example.com/MyProject.git@main")?; + let b = Url::parse("git+https://example.com/MyProject.git@main")?; + assert!(is_same_reference(&a, &b, &empty)); + + // Same repository, same tag, same subdirectory. + let a = Url::parse("git+https://example.com/MyProject.git@main#subdirectory=pkg_dir")?; + let b = Url::parse("git+https://example.com/MyProject.git@main#subdirectory=pkg_dir")?; + assert!(is_same_reference(&a, &b, &empty)); + + // Different repositories, same tag. + let a = Url::parse("git+https://example.com/MyProject.git@main")?; + let b = Url::parse("git+https://example.com/MyOtherProject.git@main")?; + assert!(!is_same_reference(&a, &b, &empty)); + + // Same repository, different tags. + let a = Url::parse("git+https://example.com/MyProject.git@main")?; + let b = Url::parse("git+https://example.com/MyProject.git@v1.0")?; + assert!(!is_same_reference(&a, &b, &empty)); + + // Same repository, same tag, different subdirectory. + let a = Url::parse("git+https://example.com/MyProject.git@main#subdirectory=pkg_dir")?; + let b = Url::parse("git+https://example.com/MyProject.git@main#subdirectory=other_dir")?; + assert!(!is_same_reference(&a, &b, &empty)); + + // Same repository, different tags, but same precise commit. + let a = Url::parse("git+https://example.com/MyProject.git@main")?; + let b = Url::parse( + "git+https://example.com/MyProject.git@164a8735b081663fede48c5041667b194da15d25", + )?; + let resolved_refs = GitResolver::default(); + resolved_refs.insert( + RepositoryReference::from(&GitUrl::try_from(Url::parse( + "https://example.com/MyProject@main", + )?)?), + GitSha::from_str("164a8735b081663fede48c5041667b194da15d25")?, + ); + assert!(is_same_reference(&a, &b, &resolved_refs)); + + // Same repository, different tags, different precise commit. + let a = Url::parse("git+https://example.com/MyProject.git@main")?; + let b = Url::parse( + "git+https://example.com/MyProject.git@164a8735b081663fede48c5041667b194da15d25", + )?; + let resolved_refs = GitResolver::default(); + resolved_refs.insert( + RepositoryReference::from(&GitUrl::try_from(Url::parse( + "https://example.com/MyProject@main", + )?)?), + GitSha::from_str("f2c9e88f3ec9526bbcec68d150b176d96a750aba")?, + ); + assert!(!is_same_reference(&a, &b, &resolved_refs)); + + Ok(()) + } } diff --git a/crates/uv-resolver/tests/resolver.rs b/crates/uv-resolver/tests/resolver.rs index 8a87ea92a..7079deea0 100644 --- a/crates/uv-resolver/tests/resolver.rs +++ b/crates/uv-resolver/tests/resolver.rs @@ -21,6 +21,7 @@ use uv_configuration::{ BuildKind, Concurrency, Constraints, NoBinary, NoBuild, Overrides, PreviewMode, SetupPyStrategy, }; use uv_distribution::DistributionDatabase; +use uv_git::GitResolver; use uv_interpreter::{find_default_interpreter, Interpreter, PythonEnvironment}; use uv_normalize::PackageName; use uv_resolver::{ @@ -45,6 +46,7 @@ struct DummyContext { cache: Cache, interpreter: Interpreter, index_locations: IndexLocations, + git: GitResolver, } impl DummyContext { @@ -53,6 +55,7 @@ impl DummyContext { cache, interpreter, index_locations: IndexLocations::default(), + git: GitResolver::default(), } } } @@ -64,6 +67,10 @@ impl BuildContext for DummyContext { &self.cache } + fn git(&self) -> &GitResolver { + &self.git + } + fn interpreter(&self) -> &Interpreter { &self.interpreter } diff --git a/crates/uv-types/Cargo.toml b/crates/uv-types/Cargo.toml index 78247b100..7ab1ffcc7 100644 --- a/crates/uv-types/Cargo.toml +++ b/crates/uv-types/Cargo.toml @@ -19,9 +19,10 @@ pep440_rs = { workspace = true } pep508_rs = { workspace = true } pypi-types = { workspace = true } uv-cache = { workspace = true } +uv-configuration = { workspace = true } +uv-git = { workspace = true } uv-interpreter = { workspace = true } uv-normalize = { workspace = true } -uv-configuration = { workspace = true } anyhow = { workspace = true } rustc-hash = { workspace = true } diff --git a/crates/uv-types/src/traits.rs b/crates/uv-types/src/traits.rs index dfab1d139..a525c2e43 100644 --- a/crates/uv-types/src/traits.rs +++ b/crates/uv-types/src/traits.rs @@ -8,6 +8,7 @@ use pep508_rs::PackageName; use pypi_types::Requirement; use uv_cache::Cache; use uv_configuration::{BuildKind, NoBinary, NoBuild, SetupPyStrategy}; +use uv_git::GitResolver; use uv_interpreter::{Interpreter, PythonEnvironment}; use crate::BuildIsolation; @@ -55,6 +56,9 @@ pub trait BuildContext { /// Return a reference to the cache. fn cache(&self) -> &Cache; + /// Return a reference to the Git resolver. + fn git(&self) -> &GitResolver; + /// All (potentially nested) source distribution builds use the same base python and can reuse /// it's metadata (e.g. wheel compatibility tags). fn interpreter(&self) -> &Interpreter; diff --git a/crates/uv/Cargo.toml b/crates/uv/Cargo.toml index a45d43a63..98a395557 100644 --- a/crates/uv/Cargo.toml +++ b/crates/uv/Cargo.toml @@ -26,6 +26,7 @@ uv-configuration = { workspace = true, features = ["clap"] } uv-dispatch = { workspace = true } uv-distribution = { workspace = true } uv-fs = { workspace = true } +uv-git = { workspace = true } uv-installer = { workspace = true } uv-interpreter = { workspace = true } uv-normalize = { workspace = true } diff --git a/crates/uv/src/commands/pip/compile.rs b/crates/uv/src/commands/pip/compile.rs index 9a579f686..4f2cdd2e8 100644 --- a/crates/uv/src/commands/pip/compile.rs +++ b/crates/uv/src/commands/pip/compile.rs @@ -27,6 +27,7 @@ use uv_configuration::{KeyringProviderType, TargetTriple}; use uv_dispatch::BuildDispatch; use uv_distribution::DistributionDatabase; use uv_fs::Simplified; +use uv_git::GitResolver; use uv_interpreter::{ find_best_interpreter, find_interpreter, InterpreterRequest, PythonEnvironment, SystemPython, VersionRequest, @@ -286,6 +287,7 @@ pub(crate) async fn pip_compile( // Read the lockfile, if present. let preferences = read_requirements_txt(output_file, &upgrade).await?; + let git = GitResolver::default(); // Resolve the flat indexes from `--find-links`. let flat_index = { @@ -316,6 +318,7 @@ pub(crate) async fn pip_compile( &index_locations, &flat_index, &source_index, + &git, &in_flight, setup_py, &config_settings, diff --git a/crates/uv/src/commands/pip/install.rs b/crates/uv/src/commands/pip/install.rs index 83a20d6c8..03712dc6e 100644 --- a/crates/uv/src/commands/pip/install.rs +++ b/crates/uv/src/commands/pip/install.rs @@ -20,6 +20,7 @@ use uv_configuration::{ use uv_configuration::{KeyringProviderType, TargetTriple}; use uv_dispatch::BuildDispatch; use uv_fs::Simplified; +use uv_git::GitResolver; use uv_installer::{SatisfiesResult, SitePackages}; use uv_interpreter::{PythonEnvironment, PythonVersion, SystemPython, Target}; use uv_normalize::PackageName; @@ -255,6 +256,7 @@ pub(crate) async fn pip_install( // When resolving, don't take any external preferences into account. let preferences = Vec::default(); + let git = GitResolver::default(); // Incorporate any index locations from the provided sources. let index_locations = @@ -308,6 +310,7 @@ pub(crate) async fn pip_install( &index_locations, &flat_index, &index, + &git, &in_flight, setup_py, config_settings, @@ -387,6 +390,7 @@ pub(crate) async fn pip_install( &index_locations, &flat_index, &index, + &git, &in_flight, setup_py, config_settings, diff --git a/crates/uv/src/commands/pip/sync.rs b/crates/uv/src/commands/pip/sync.rs index b902dabae..f625967e3 100644 --- a/crates/uv/src/commands/pip/sync.rs +++ b/crates/uv/src/commands/pip/sync.rs @@ -19,6 +19,7 @@ use uv_configuration::{ use uv_configuration::{KeyringProviderType, TargetTriple}; use uv_dispatch::BuildDispatch; use uv_fs::Simplified; +use uv_git::GitResolver; use uv_installer::SitePackages; use uv_interpreter::{PythonEnvironment, PythonVersion, SystemPython, Target}; use uv_requirements::{RequirementsSource, RequirementsSpecification}; @@ -251,6 +252,10 @@ pub(crate) async fn pip_sync( // Track in-flight downloads, builds, etc., across resolutions. let in_flight = InFlight::default(); + // When resolving, don't take any external preferences into account. + let preferences = Vec::default(); + let git = GitResolver::default(); + // Create a build dispatch for resolution. let resolve_dispatch = BuildDispatch::new( &client, @@ -259,6 +264,7 @@ pub(crate) async fn pip_sync( &index_locations, &flat_index, &index, + &git, &in_flight, setup_py, config_settings, @@ -274,9 +280,6 @@ pub(crate) async fn pip_sync( // Determine the set of installed packages. let site_packages = SitePackages::from_executable(&venv)?; - // When resolving, don't take any external preferences into account. - let preferences = Vec::default(); - let options = OptionsBuilder::new() .resolution_mode(resolution_mode) .prerelease_mode(prerelease_mode) @@ -336,6 +339,7 @@ pub(crate) async fn pip_sync( &index_locations, &flat_index, &index, + &git, &in_flight, setup_py, config_settings, diff --git a/crates/uv/src/commands/project/lock.rs b/crates/uv/src/commands/project/lock.rs index 2c930df5c..2b88f2606 100644 --- a/crates/uv/src/commands/project/lock.rs +++ b/crates/uv/src/commands/project/lock.rs @@ -11,6 +11,7 @@ use uv_configuration::{ }; use uv_dispatch::BuildDispatch; use uv_distribution::ProjectWorkspace; +use uv_git::GitResolver; use uv_interpreter::PythonEnvironment; use uv_requirements::upgrade::read_lockfile; use uv_resolver::{ExcludeNewer, FlatIndex, InMemoryIndex, Lock, OptionsBuilder}; @@ -104,6 +105,7 @@ pub(super) async fn do_lock( let config_settings = ConfigSettings::default(); let extras = ExtrasSpecification::default(); let flat_index = FlatIndex::default(); + let git = GitResolver::default(); let in_flight = InFlight::default(); let index = InMemoryIndex::default(); let index_locations = IndexLocations::default(); @@ -127,6 +129,7 @@ pub(super) async fn do_lock( &index_locations, &flat_index, &index, + &git, &in_flight, setup_py, &config_settings, diff --git a/crates/uv/src/commands/project/mod.rs b/crates/uv/src/commands/project/mod.rs index ca85ad15f..28381d689 100644 --- a/crates/uv/src/commands/project/mod.rs +++ b/crates/uv/src/commands/project/mod.rs @@ -16,6 +16,7 @@ use uv_configuration::{ use uv_dispatch::BuildDispatch; use uv_distribution::ProjectWorkspace; use uv_fs::Simplified; +use uv_git::GitResolver; use uv_installer::{SatisfiesResult, SitePackages}; use uv_interpreter::{find_default_interpreter, PythonEnvironment}; use uv_requirements::{RequirementsSource, RequirementsSpecification}; @@ -167,6 +168,7 @@ pub(crate) async fn update_environment( let dry_run = false; let extras = ExtrasSpecification::default(); let flat_index = FlatIndex::default(); + let git = GitResolver::default(); let hasher = HashStrategy::default(); let in_flight = InFlight::default(); let index = InMemoryIndex::default(); @@ -188,6 +190,7 @@ pub(crate) async fn update_environment( &index_locations, &flat_index, &index, + &git, &in_flight, setup_py, &config_settings, @@ -245,6 +248,7 @@ pub(crate) async fn update_environment( &index_locations, &flat_index, &index, + &git, &in_flight, setup_py, &config_settings, diff --git a/crates/uv/src/commands/project/sync.rs b/crates/uv/src/commands/project/sync.rs index d4ba2e0e5..3a9c6ff2e 100644 --- a/crates/uv/src/commands/project/sync.rs +++ b/crates/uv/src/commands/project/sync.rs @@ -10,6 +10,7 @@ use uv_configuration::{ }; use uv_dispatch::BuildDispatch; use uv_distribution::ProjectWorkspace; +use uv_git::GitResolver; use uv_installer::SitePackages; use uv_interpreter::PythonEnvironment; use uv_resolver::{FlatIndex, InMemoryIndex, Lock}; @@ -87,6 +88,7 @@ pub(super) async fn do_sync( let config_settings = ConfigSettings::default(); let dry_run = false; let flat_index = FlatIndex::default(); + let git = GitResolver::default(); let hasher = HashStrategy::default(); let in_flight = InFlight::default(); let index = InMemoryIndex::default(); @@ -105,6 +107,7 @@ pub(super) async fn do_sync( &index_locations, &flat_index, &index, + &git, &in_flight, setup_py, &config_settings, diff --git a/crates/uv/src/commands/venv.rs b/crates/uv/src/commands/venv.rs index 61700664d..5e13f5fc2 100644 --- a/crates/uv/src/commands/venv.rs +++ b/crates/uv/src/commands/venv.rs @@ -19,6 +19,7 @@ use uv_configuration::{Concurrency, KeyringProviderType, PreviewMode}; use uv_configuration::{ConfigSettings, IndexStrategy, NoBinary, NoBuild, SetupPyStrategy}; use uv_dispatch::BuildDispatch; use uv_fs::Simplified; +use uv_git::GitResolver; use uv_interpreter::{ find_default_interpreter, find_interpreter, InterpreterRequest, SourceSelector, }; @@ -198,6 +199,7 @@ async fn venv_impl( // Create a shared in-memory index. let index = InMemoryIndex::default(); + let git = GitResolver::default(); // Track in-flight downloads, builds, etc., across resolutions. let in_flight = InFlight::default(); @@ -214,6 +216,7 @@ async fn venv_impl( index_locations, &flat_index, &index, + &git, &in_flight, SetupPyStrategy::default(), &config_settings,