Python
/
uv
mirror of https://github.com/astral-sh/uv
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove fast path from `uv-git` fetch (#16607) 

## Summary

Now that we perform this fast-path in
`crates/uv-distribution/src/source/mod.rs`, I _think_ the fast-path here
is no longer used? In my testing, we only actually took this path when
the fast-path _already_ failed (and thus it would fail again, wasting
time).
This commit is contained in:
Charlie Marsh 2025-11-06 23:18:05 -05:00 committed by GitHub
parent e5c10cd45d
commit 4740d267e9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 31 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
crates/uv-distribution/src/source/mod.rs
View File

@ -1542,10 +1542,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.git() .git()
.fetch( .fetch(
resource.git, resource.git,
client
.unmanaged
.uncached_client(resource.git.repository())
.clone(),
client.unmanaged.disable_ssl(resource.git.repository()), client.unmanaged.disable_ssl(resource.git.repository()),
client.unmanaged.connectivity() == Connectivity::Offline, client.unmanaged.connectivity() == Connectivity::Offline,
self.build_context.cache().bucket(CacheBucket::Git), self.build_context.cache().bucket(CacheBucket::Git),
@ -1746,10 +1742,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.git() .git()
.fetch( .fetch(
resource.git, resource.git,
client
.unmanaged
.uncached_client(resource.git.repository())
.clone(),
client.unmanaged.disable_ssl(resource.git.repository()), client.unmanaged.disable_ssl(resource.git.repository()),
client.unmanaged.connectivity() == Connectivity::Offline, client.unmanaged.connectivity() == Connectivity::Offline,
self.build_context.cache().bucket(CacheBucket::Git), self.build_context.cache().bucket(CacheBucket::Git),
@ -1958,23 +1950,23 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
&self, &self,
source: &BuildableSource<'_>, source: &BuildableSource<'_>,
client: &ManagedClient<'_>, client: &ManagedClient<'_>,
) -> Result<(), Error> { ) -> Result<Option<GitOid>, Error> {
let git = match source { let git = match source {
BuildableSource::Dist(SourceDist::Git(source)) => &*source.git, BuildableSource::Dist(SourceDist::Git(source)) => &*source.git,
BuildableSource::Url(SourceUrl::Git(source)) => source.git, BuildableSource::Url(SourceUrl::Git(source)) => source.git,
_ => { _ => {
return Ok(()); return Ok(None);
} }
}; };
// If the URL is already precise, return it. // If the URL is already precise, return it.
if self.build_context.git().get_precise(git).is_some() { if let Some(precise) = self.build_context.git().get_precise(git) {
debug!("Precise commit already known: {source}"); debug!("Precise commit already known: {source}");
return Ok(()); return Ok(Some(precise));
} }
// If this is GitHub URL, attempt to resolve to a precise commit using the GitHub API. // If this is GitHub URL, attempt to resolve to a precise commit using the GitHub API.
if self if let Some(precise) = self
.build_context .build_context
.git() .git()
.github_fast_path( .github_fast_path(
@ -1985,18 +1977,17 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.raw_client(), .raw_client(),
) )
.await? .await?
.is_some()
{ {
debug!("Resolved to precise commit via GitHub fast path: {source}"); debug!("Resolved to precise commit via GitHub fast path: {source}");
return Ok(()); return Ok(Some(precise));
} }
// Otherwise, fetch the Git repository. // Otherwise, fetch the Git repository.
self.build_context let fetch = self
.build_context
.git() .git()
.fetch( .fetch(
git, git,
client.unmanaged.uncached_client(git.repository()).clone(),
client.unmanaged.disable_ssl(git.repository()), client.unmanaged.disable_ssl(git.repository()),
client.unmanaged.connectivity() == Connectivity::Offline, client.unmanaged.connectivity() == Connectivity::Offline,
self.build_context.cache().bucket(CacheBucket::Git), self.build_context.cache().bucket(CacheBucket::Git),
@ -2006,7 +1997,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
) )
.await?; .await?;
Ok(()) Ok(fetch.git().precise())
} }
/// Fetch static [`ResolutionMetadata`] from a GitHub repository, if possible. /// Fetch static [`ResolutionMetadata`] from a GitHub repository, if possible.

182
crates/uv-git/src/git.rs
View File

@ -1,7 +1,6 @@
//! Git support is derived from Cargo's implementation. //! Git support is derived from Cargo's implementation.
//! Cargo is dual-licensed under either Apache 2.0 or MIT, at the user's choice. //! Cargo is dual-licensed under either Apache 2.0 or MIT, at the user's choice.
//! Source: <https://github.com/rust-lang/cargo/blob/23eb492cf920ce051abfc56bbaf838514dc8365c/src/cargo/sources/git/utils.rs> //! Source: <https://github.com/rust-lang/cargo/blob/23eb492cf920ce051abfc56bbaf838514dc8365c/src/cargo/sources/git/utils.rs>
use std::env;
use std::fmt::Display; use std::fmt::Display;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::str::{self}; use std::str::{self};
@ -9,18 +8,13 @@ use std::sync::LazyLock;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use cargo_util::{ProcessBuilder, paths}; use cargo_util::{ProcessBuilder, paths};
use reqwest::StatusCode;
use reqwest_middleware::ClientWithMiddleware;
use tracing::{debug, warn}; use tracing::{debug, warn};
use url::Url; use url::Url;
use uv_fs::Simplified; use uv_fs::Simplified;
use uv_git_types::{GitHubRepository, GitOid, GitReference}; use uv_git_types::{GitOid, GitReference};
use uv_redacted::DisplaySafeUrl; use uv_redacted::DisplaySafeUrl;
use uv_static::EnvVars; use uv_static::EnvVars;
use uv_version::version;
use crate::rate_limit::{GITHUB_RATE_LIMIT_STATUS, is_github_rate_limited};
/// A file indicates that if present, `git reset` has been done and a repo /// A file indicates that if present, `git reset` has been done and a repo
/// checkout is ready to go. See [`GitCheckout::reset`] for why we need this. /// checkout is ready to go. See [`GitCheckout::reset`] for why we need this.
@ -235,24 +229,16 @@ impl GitRemote {
db: Option<GitDatabase>, db: Option<GitDatabase>,
reference: &GitReference, reference: &GitReference,
locked_rev: Option<GitOid>, locked_rev: Option<GitOid>,
client: &ClientWithMiddleware,
disable_ssl: bool, disable_ssl: bool,
offline: bool, offline: bool,
) -> Result<(GitDatabase, GitOid)> { ) -> Result<(GitDatabase, GitOid)> {
let reference = locked_rev let reference = locked_rev
.map(ReferenceOrOid::Oid) .map(ReferenceOrOid::Oid)
.unwrap_or(ReferenceOrOid::Reference(reference)); .unwrap_or(ReferenceOrOid::Reference(reference));
let enable_lfs_fetch = env::var(EnvVars::UV_GIT_LFS).is_ok(); let enable_lfs_fetch = std::env::var(EnvVars::UV_GIT_LFS).is_ok();
if let Some(mut db) = db { if let Some(mut db) = db {
fetch( fetch(&mut db.repo, &self.url, reference, disable_ssl, offline)
&mut db.repo,
&self.url,
reference,
client,
disable_ssl,
offline,
)
.with_context(|| format!("failed to fetch into: {}", into.user_display()))?; .with_context(|| format!("failed to fetch into: {}", into.user_display()))?;
let resolved_commit_hash = match locked_rev { let resolved_commit_hash = match locked_rev {
@ -280,14 +266,7 @@ impl GitRemote {
fs_err::create_dir_all(into)?; fs_err::create_dir_all(into)?;
let mut repo = GitRepository::init(into)?; let mut repo = GitRepository::init(into)?;
fetch( fetch(&mut repo, &self.url, reference, disable_ssl, offline)
&mut repo,
&self.url,
reference,
client,
disable_ssl,
offline,
)
.with_context(|| format!("failed to clone into: {}", into.user_display()))?; .with_context(|| format!("failed to clone into: {}", into.user_display()))?;
let rev = match locked_rev { let rev = match locked_rev {
Some(rev) => rev, Some(rev) => rev,
@ -461,18 +440,22 @@ fn fetch(
repo: &mut GitRepository, repo: &mut GitRepository,
remote_url: &Url, remote_url: &Url,
reference: ReferenceOrOid<'_>, reference: ReferenceOrOid<'_>,
client: &ClientWithMiddleware,
disable_ssl: bool, disable_ssl: bool,
offline: bool, offline: bool,
) -> Result<()> { ) -> Result<()> {
let oid_to_fetch = match github_fast_path(repo, remote_url, reference, client) { let oid_to_fetch = if let ReferenceOrOid::Oid(rev) = reference {
Ok(FastPathRev::UpToDate) => return Ok(()), let local_object = reference.resolve(repo).ok();
Ok(FastPathRev::NeedsFetch(rev)) => Some(rev), if let Some(local_object) = local_object {
Ok(FastPathRev::Indeterminate) => None, if rev == local_object {
Err(e) => { return Ok(());
debug!("Failed to check GitHub {:?}", e);
None
} }
}
// If we know the reference is a full commit hash, we can just return it without
// querying GitHub.
Some(rev)
} else {
None
}; };
// Translate the reference desired here into an actual list of refspecs // Translate the reference desired here into an actual list of refspecs
@ -705,139 +688,6 @@ fn fetch_lfs(
Ok(()) Ok(())
} }
/// The result of GitHub fast path check. See [`github_fast_path`] for more.
enum FastPathRev {
/// The local rev (determined by `reference.resolve(repo)`) is already up to
/// date with what this rev resolves to on GitHub's server.
UpToDate,
/// The following SHA must be fetched in order for the local rev to become
/// up-to-date.
NeedsFetch(GitOid),
/// Don't know whether local rev is up-to-date. We'll fetch _all_ branches
/// and tags from the server and see what happens.
Indeterminate,
}
/// Attempts GitHub's special fast path for testing if we've already got an
/// up-to-date copy of the repository.
///
/// Updating the index is done pretty regularly so we want it to be as fast as
/// possible. For registries hosted on GitHub (like the crates.io index) there's
/// a fast path available to use[^1] to tell us that there's no updates to be
/// made.
///
/// Note that this function should never cause an actual failure because it's
/// just a fast path. As a result, a caller should ignore `Err` returned from
/// this function and move forward on the normal path.
///
/// [^1]: <https://developer.github.com/v3/repos/commits/#get-the-sha-1-of-a-commit-reference>
fn github_fast_path(
git: &mut GitRepository,
url: &Url,
reference: ReferenceOrOid<'_>,
client: &ClientWithMiddleware,
) -> Result<FastPathRev> {
let Some(GitHubRepository { owner, repo }) = GitHubRepository::parse(url) else {
return Ok(FastPathRev::Indeterminate);
};
let local_object = reference.resolve(git).ok();
let github_branch_name = match reference {
ReferenceOrOid::Reference(GitReference::DefaultBranch) => "HEAD",
ReferenceOrOid::Reference(GitReference::Branch(branch)) => branch,
ReferenceOrOid::Reference(GitReference::Tag(tag)) => tag,
ReferenceOrOid::Reference(GitReference::BranchOrTag(branch_or_tag)) => branch_or_tag,
ReferenceOrOid::Reference(GitReference::NamedRef(rev)) => rev,
ReferenceOrOid::Reference(GitReference::BranchOrTagOrCommit(rev)) => {
// `revparse_single` (used by `resolve`) is the only way to turn
// short hash -> long hash, but it also parses other things,
// like branch and tag names, which might coincidentally be
// valid hex.
//
// We only return early if `rev` is a prefix of the object found
// by `revparse_single`. Don't bother talking to GitHub in that
// case, since commit hashes are permanent. If a commit with the
// requested hash is already present in the local clone, its
// contents must be the same as what is on the server for that
// hash.
//
// If `rev` is not found locally by `revparse_single`, we'll
// need GitHub to resolve it and get a hash. If `rev` is found
// but is not a short hash of the found object, it's probably a
// branch and we also need to get a hash from GitHub, in case
// the branch has moved.
if let Some(ref local_object) = local_object {
if is_short_hash_of(rev, *local_object) {
return Ok(FastPathRev::UpToDate);
}
}
rev
}
ReferenceOrOid::Oid(rev) => {
debug!("Skipping GitHub fast path; full commit hash provided: {rev}");
if let Some(local_object) = local_object {
if rev == local_object {
return Ok(FastPathRev::UpToDate);
}
}
// If we know the reference is a full commit hash, we can just return it without
// querying GitHub.
return Ok(FastPathRev::NeedsFetch(rev));
}
};
// Check if we're rate-limited by GitHub before determining the FastPathRev
if GITHUB_RATE_LIMIT_STATUS.is_active() {
debug!("Skipping GitHub fast path attempt for: {url} (rate-limited)");
return Ok(FastPathRev::Indeterminate);
}
let base_url = std::env::var(EnvVars::UV_GITHUB_FAST_PATH_URL)
.unwrap_or("https://api.github.com/repos".to_owned());
let url = format!("{base_url}/{owner}/{repo}/commits/{github_branch_name}");
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
runtime.block_on(async move {
debug!("Attempting GitHub fast path for: {url}");
let mut request = client.get(&url);
request = request.header("Accept", "application/vnd.github.3.sha");
request = request.header(
"User-Agent",
format!("uv/{} (+https://github.com/astral-sh/uv)", version()),
);
if let Some(local_object) = local_object {
request = request.header("If-None-Match", local_object.to_string());
}
let response = request.send().await?;
if is_github_rate_limited(&response) {
// Mark that we are being rate-limited by GitHub
GITHUB_RATE_LIMIT_STATUS.activate();
}
// GitHub returns a 404 if the repository does not exist, and a 422 if it exists but GitHub
// is unable to resolve the requested revision.
response.error_for_status_ref()?;
let response_code = response.status();
if response_code == StatusCode::NOT_MODIFIED {
Ok(FastPathRev::UpToDate)
} else if response_code == StatusCode::OK {
let oid_to_fetch = response.text().await?.parse()?;
Ok(FastPathRev::NeedsFetch(oid_to_fetch))
} else {
Ok(FastPathRev::Indeterminate)
}
})
}
/// Whether `rev` is a shorter hash of `oid`. /// Whether `rev` is a shorter hash of `oid`.
fn is_short_hash_of(rev: &str, oid: GitOid) -> bool { fn is_short_hash_of(rev: &str, oid: GitOid) -> bool {
let long_hash = oid.to_string(); let long_hash = oid.to_string();

6
crates/uv-git/src/resolver.rs
View File

@ -49,6 +49,7 @@ impl GitResolver {
self.0.get(reference) self.0.get(reference)
} }
/// Return the [`GitOid`] for the given [`GitUrl`], if it is already known.
pub fn get_precise(&self, url: &GitUrl) -> Option<GitOid> { pub fn get_precise(&self, url: &GitUrl) -> Option<GitOid> {
// If the URL is already precise, return it. // If the URL is already precise, return it.
if let Some(precise) = url.precise() { if let Some(precise) = url.precise() {
@ -143,7 +144,6 @@ impl GitResolver {
pub async fn fetch( pub async fn fetch(
&self, &self,
url: &GitUrl, url: &GitUrl,
client: impl Into<ClientWithMiddleware>,
disable_ssl: bool, disable_ssl: bool,
offline: bool, offline: bool,
cache: PathBuf, cache: PathBuf,
@ -175,9 +175,9 @@ impl GitResolver {
// Fetch the Git repository. // Fetch the Git repository.
let source = if let Some(reporter) = reporter { let source = if let Some(reporter) = reporter {
GitSource::new(url.as_ref().clone(), client, cache, offline).with_reporter(reporter) GitSource::new(url.as_ref().clone(), cache, offline).with_reporter(reporter)
} else { } else {
GitSource::new(url.as_ref().clone(), client, cache, offline) GitSource::new(url.as_ref().clone(), cache, offline)
}; };
// If necessary, disable SSL. // If necessary, disable SSL.

12
crates/uv-git/src/source.rs
View File

@ -7,7 +7,6 @@ use std::path::{Path, PathBuf};
use std::sync::Arc; use std::sync::Arc;
use anyhow::Result; use anyhow::Result;
use reqwest_middleware::ClientWithMiddleware;
use tracing::{debug, instrument}; use tracing::{debug, instrument};
use uv_cache_key::{RepositoryUrl, cache_digest}; use uv_cache_key::{RepositoryUrl, cache_digest};
@ -21,8 +20,6 @@ use crate::git::{GitDatabase, GitRemote};
pub struct GitSource { pub struct GitSource {
/// The Git reference from the manifest file. /// The Git reference from the manifest file.
git: GitUrl, git: GitUrl,
/// The HTTP client to use for fetching.
client: ClientWithMiddleware,
/// Whether to disable SSL verification. /// Whether to disable SSL verification.
disable_ssl: bool, disable_ssl: bool,
/// Whether to operate without network connectivity. /// Whether to operate without network connectivity.
@ -35,17 +32,11 @@ pub struct GitSource {
impl GitSource { impl GitSource {
/// Initialize a [`GitSource`] with the given Git URL, HTTP client, and cache path. /// Initialize a [`GitSource`] with the given Git URL, HTTP client, and cache path.
pub fn new( pub fn new(git: GitUrl, cache: impl Into<PathBuf>, offline: bool) -> Self {
git: GitUrl,
client: impl Into<ClientWithMiddleware>,
cache: impl Into<PathBuf>,
offline: bool,
) -> Self {
Self { Self {
git, git,
disable_ssl: false, disable_ssl: false,
offline, offline,
client: client.into(),
cache: cache.into(), cache: cache.into(),
reporter: None, reporter: None,
} }
@ -132,7 +123,6 @@ impl GitSource {
maybe_db, maybe_db,
self.git.reference(), self.git.reference(),
self.git.precise(), self.git.precise(),
&self.client,
self.disable_ssl, self.disable_ssl,
self.offline, self.offline,
)?; )?;