Python
/
uv
mirror of https://github.com/astral-sh/uv
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove source distribution filename from cache (#8907) 

## Summary

In the example outlined in https://github.com/astral-sh/uv/issues/8884,
this removes an unnecessary `jupyter_contrib_nbextensions-0.7.0.tar.gz`
segment (replacing it with `src`), thereby saving 39 characters and
getting that build working on my Windows machine.

This should _not_ require a version bump because we already have logic
in place to "heal" partial cache entries that lack an unzipped
distribution.

Closes https://github.com/astral-sh/uv/issues/8884.

Closes https://github.com/astral-sh/uv/issues/7376.
This commit is contained in:
Charlie Marsh 2024-11-07 19:50:06 -05:00 committed by GitHub
parent 8a1b581d07
commit 88033610b5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 18 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
crates/uv-distribution/src/source/mod.rs
View File

@ -29,7 +29,7 @@ use uv_configuration::{BuildKind, BuildOutput, SourceStrategy};
use uv_distribution_filename::{SourceDistExtension, WheelFilename}; use uv_distribution_filename::{SourceDistExtension, WheelFilename};
use uv_distribution_types::{ use uv_distribution_types::{
BuildableSource, DirectorySourceUrl, FileLocation, GitSourceUrl, HashPolicy, Hashed, BuildableSource, DirectorySourceUrl, FileLocation, GitSourceUrl, HashPolicy, Hashed,
PathSourceUrl, RemoteSource, SourceDist, SourceUrl, PathSourceUrl, SourceDist, SourceUrl,
}; };
use uv_extract::hash::Hasher; use uv_extract::hash::Hasher;
use uv_fs::{rename_with_retry, write_atomic, LockedFile}; use uv_fs::{rename_with_retry, write_atomic, LockedFile};
@ -58,6 +58,9 @@ pub(crate) const LOCAL_REVISION: &str = "revision.rev";
/// The name of the file that contains the cached distribution metadata, encoded via `MsgPack`. /// The name of the file that contains the cached distribution metadata, encoded via `MsgPack`.
pub(crate) const METADATA: &str = "metadata.msgpack"; pub(crate) const METADATA: &str = "metadata.msgpack";
/// The directory within each entry under which to store the unpacked source distribution.
pub(crate) const SOURCE: &str = "src";
impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
/// Initialize a [`SourceDistributionBuilder`] from a [`BuildContext`]. /// Initialize a [`SourceDistributionBuilder`] from a [`BuildContext`].
pub(crate) fn new(build_context: &'a T) -> Self { pub(crate) fn new(build_context: &'a T) -> Self {
@ -125,7 +128,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
self.url( self.url(
source, source,
&dist.file.filename,
&url, &url,
&cache_shard, &cache_shard,
None, None,
@ -138,8 +140,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await? .await?
} }
BuildableSource::Dist(SourceDist::DirectUrl(dist)) => { BuildableSource::Dist(SourceDist::DirectUrl(dist)) => {
let filename = dist.filename().expect("Distribution must have a filename");
// For direct URLs, cache directly under the hash of the URL itself. // For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = self.build_context.cache().shard( let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions, CacheBucket::SourceDistributions,
@ -148,7 +148,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
self.url( self.url(
source, source,
&filename,
&dist.url, &dist.url,
&cache_shard, &cache_shard,
dist.subdirectory.as_deref(), dist.subdirectory.as_deref(),
@ -186,11 +185,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await? .await?
} }
BuildableSource::Url(SourceUrl::Direct(resource)) => { BuildableSource::Url(SourceUrl::Direct(resource)) => {
let filename = resource
.url
.filename()
.expect("Distribution must have a filename");
// For direct URLs, cache directly under the hash of the URL itself. // For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = self.build_context.cache().shard( let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions, CacheBucket::SourceDistributions,
@ -199,7 +193,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
self.url( self.url(
source, source,
&filename,
resource.url, resource.url,
&cache_shard, &cache_shard,
resource.subdirectory, resource.subdirectory,
@ -281,22 +274,11 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await; .await;
} }
self.url_metadata( self.url_metadata(source, &url, &cache_shard, None, dist.ext, hashes, client)
source,
&dist.file.filename,
&url,
&cache_shard,
None,
dist.ext,
hashes,
client,
)
.boxed_local() .boxed_local()
.await? .await?
} }
BuildableSource::Dist(SourceDist::DirectUrl(dist)) => { BuildableSource::Dist(SourceDist::DirectUrl(dist)) => {
let filename = dist.filename().expect("Distribution must have a filename");
// For direct URLs, cache directly under the hash of the URL itself. // For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = self.build_context.cache().shard( let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions, CacheBucket::SourceDistributions,
@ -305,7 +287,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
self.url_metadata( self.url_metadata(
source, source,
&filename,
&dist.url, &dist.url,
&cache_shard, &cache_shard,
dist.subdirectory.as_deref(), dist.subdirectory.as_deref(),
@ -336,11 +317,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await? .await?
} }
BuildableSource::Url(SourceUrl::Direct(resource)) => { BuildableSource::Url(SourceUrl::Direct(resource)) => {
let filename = resource
.url
.filename()
.expect("Distribution must have a filename");
// For direct URLs, cache directly under the hash of the URL itself. // For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = self.build_context.cache().shard( let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions, CacheBucket::SourceDistributions,
@ -349,7 +325,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
self.url_metadata( self.url_metadata(
source, source,
&filename,
resource.url, resource.url,
&cache_shard, &cache_shard,
resource.subdirectory, resource.subdirectory,
@ -403,7 +378,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
async fn url<'data>( async fn url<'data>(
&self, &self,
source: &BuildableSource<'data>, source: &BuildableSource<'data>,
filename: &'data str,
url: &'data Url, url: &'data Url,
cache_shard: &CacheShard, cache_shard: &CacheShard,
subdirectory: Option<&'data Path>, subdirectory: Option<&'data Path>,
@ -416,7 +390,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Fetch the revision for the source distribution. // Fetch the revision for the source distribution.
let revision = self let revision = self
.url_revision(source, filename, ext, url, cache_shard, hashes, client) .url_revision(source, ext, url, cache_shard, hashes, client)
.await?; .await?;
// Before running the build, check that the hashes match. // Before running the build, check that the hashes match.
@ -431,7 +405,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Scope all operations to the revision. Within the revision, there's no need to check for // Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself. // freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id()); let cache_shard = cache_shard.shard(revision.id());
let source_dist_entry = cache_shard.entry(filename); let source_dist_entry = cache_shard.entry(SOURCE);
// If there are build settings, we need to scope to a cache shard. // If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings(); let config_settings = self.build_context.config_settings();
@ -452,7 +426,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
} else { } else {
self.heal_url_revision( self.heal_url_revision(
source, source,
filename,
ext, ext,
url, url,
&source_dist_entry, &source_dist_entry,
@ -507,7 +480,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
async fn url_metadata<'data>( async fn url_metadata<'data>(
&self, &self,
source: &BuildableSource<'data>, source: &BuildableSource<'data>,
filename: &'data str,
url: &'data Url, url: &'data Url,
cache_shard: &CacheShard, cache_shard: &CacheShard,
subdirectory: Option<&'data Path>, subdirectory: Option<&'data Path>,
@ -519,7 +491,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Fetch the revision for the source distribution. // Fetch the revision for the source distribution.
let revision = self let revision = self
.url_revision(source, filename, ext, url, cache_shard, hashes, client) .url_revision(source, ext, url, cache_shard, hashes, client)
.await?; .await?;
// Before running the build, check that the hashes match. // Before running the build, check that the hashes match.
@ -534,7 +506,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Scope all operations to the revision. Within the revision, there's no need to check for // Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself. // freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id()); let cache_shard = cache_shard.shard(revision.id());
let source_dist_entry = cache_shard.entry(filename); let source_dist_entry = cache_shard.entry(SOURCE);
// If the metadata is static, return it. // If the metadata is static, return it.
if let Some(metadata) = if let Some(metadata) =
@ -562,7 +534,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
} else { } else {
self.heal_url_revision( self.heal_url_revision(
source, source,
filename,
ext, ext,
url, url,
&source_dist_entry, &source_dist_entry,
@ -644,7 +615,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
async fn url_revision( async fn url_revision(
&self, &self,
source: &BuildableSource<'_>, source: &BuildableSource<'_>,
filename: &str,
ext: SourceDistExtension, ext: SourceDistExtension,
url: &Url, url: &Url,
cache_shard: &CacheShard, cache_shard: &CacheShard,
@ -670,9 +640,9 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Download the source distribution. // Download the source distribution.
debug!("Downloading source distribution: {source}"); debug!("Downloading source distribution: {source}");
let entry = cache_shard.shard(revision.id()).entry(filename); let entry = cache_shard.shard(revision.id()).entry(SOURCE);
let hashes = self let hashes = self
.download_archive(response, source, filename, ext, entry.path(), hashes) .download_archive(response, source, ext, entry.path(), hashes)
.await?; .await?;
Ok(revision.with_hashes(hashes)) Ok(revision.with_hashes(hashes))
@ -743,7 +713,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Scope all operations to the revision. Within the revision, there's no need to check for // Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself. // freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id()); let cache_shard = cache_shard.shard(revision.id());
let source_entry = cache_shard.entry("source"); let source_entry = cache_shard.entry(SOURCE);
// If there are build settings, we need to scope to a cache shard. // If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings(); let config_settings = self.build_context.config_settings();
@ -832,7 +802,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Scope all operations to the revision. Within the revision, there's no need to check for // Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself. // freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id()); let cache_shard = cache_shard.shard(revision.id());
let source_entry = cache_shard.entry("source"); let source_entry = cache_shard.entry(SOURCE);
// If the metadata is static, return it. // If the metadata is static, return it.
if let Some(metadata) = if let Some(metadata) =
@ -957,7 +927,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Unzip the archive to a temporary directory. // Unzip the archive to a temporary directory.
debug!("Unpacking source distribution: {source}"); debug!("Unpacking source distribution: {source}");
let entry = cache_shard.shard(revision.id()).entry("source"); let entry = cache_shard.shard(revision.id()).entry(SOURCE);
let hashes = self let hashes = self
.persist_archive(&resource.path, resource.ext, entry.path(), hashes) .persist_archive(&resource.path, resource.ext, entry.path(), hashes)
.await?; .await?;
@ -1568,7 +1538,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
async fn heal_url_revision( async fn heal_url_revision(
&self, &self,
source: &BuildableSource<'_>, source: &BuildableSource<'_>,
filename: &str,
ext: SourceDistExtension, ext: SourceDistExtension,
url: &Url, url: &Url,
entry: &CacheEntry, entry: &CacheEntry,
@ -1581,7 +1550,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
let download = |response| { let download = |response| {
async { async {
let hashes = self let hashes = self
.download_archive(response, source, filename, ext, entry.path(), hashes) .download_archive(response, source, ext, entry.path(), hashes)
.await?; .await?;
if hashes != revision.hashes() { if hashes != revision.hashes() {
return Err(Error::CacheHeal(source.to_string())); return Err(Error::CacheHeal(source.to_string()));
@ -1610,7 +1579,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
&self, &self,
response: Response, response: Response,
source: &BuildableSource<'_>, source: &BuildableSource<'_>,
filename: &str,
ext: SourceDistExtension, ext: SourceDistExtension,
target: &Path, target: &Path,
hashes: HashPolicy<'_>, hashes: HashPolicy<'_>,
@ -1632,7 +1600,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
let mut hasher = uv_extract::hash::HashReader::new(reader.compat(), &mut hashers); let mut hasher = uv_extract::hash::HashReader::new(reader.compat(), &mut hashers);
// Download and unzip the source distribution into a temporary directory. // Download and unzip the source distribution into a temporary directory.
let span = info_span!("download_source_dist", filename = filename, source_dist = %source); let span = info_span!("download_source_dist", source_dist = %source);
uv_extract::stream::archive(&mut hasher, ext, temp_dir.path()).await?; uv_extract::stream::archive(&mut hasher, ext, temp_dir.path()).await?;
drop(span); drop(span);