Python
/
uv
mirror of https://github.com/astral-sh/uv
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Avoid persisting manifest data in standalone file (#1044) 

## Summary

This PR gets rid of the manifest that we store for source distributions.
Historically, that manifest included the source distribution metadata,
plus a list of built wheels.

The problem with the manifest is that it duplicates state, since we now
have to look at both the manifest and the filesystem to understand the
cache state. Instead, I think we should treat the cache as the source of
truth, and get rid of the duplicated state in the manifest.

Now, we store the manifest (which is merely used to check for cache
freshness -- in future PRs, I will repurpose it though, so I left it
around), then the distribution metadata as its own file, then any
distributions in the same directory. When we want to see if there are
any valid distributions, we `readdir` on the directory. This is also
much more consistent with how the install plan works.
This commit is contained in:
Charlie Marsh 2024-01-23 14:46:48 -05:00 committed by GitHub
parent 19c5cc8aba
commit e32027e384
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 148 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
crates/puffin-cache/src/lib.rs
View File

@ -294,9 +294,9 @@ pub enum CacheBucket {
/// directories in the cache. /// directories in the cache.
/// ///
/// Cache structure: /// Cache structure:
/// * `built-wheels-v0/pypi/foo/34a17436ed1e9669/{metadata.msgpack, foo-1.0.0.zip, foo-1.0.0-py3-none-any.whl, ...other wheels}` /// * `built-wheels-v0/pypi/foo/34a17436ed1e9669/{manifest.msgpack, metadata.msgpack, foo-1.0.0.zip, foo-1.0.0-py3-none-any.whl, ...other wheels}`
/// * `built-wheels-v0/<digest(index-url)>/foo/foo-1.0.0.zip/{metadata.msgpack, foo-1.0.0-py3-none-any.whl, ...other wheels}` /// * `built-wheels-v0/<digest(index-url)>/foo/foo-1.0.0.zip/{manifest.msgpack, metadata.msgpack, foo-1.0.0-py3-none-any.whl, ...other wheels}`
/// * `built-wheels-v0/url/<digest(url)>/foo/foo-1.0.0.zip/{metadata.msgpack, foo-1.0.0-py3-none-any.whl, ...other wheels}` /// * `built-wheels-v0/url/<digest(url)>/foo/foo-1.0.0.zip/{manifest.msgpack, metadata.msgpack, foo-1.0.0-py3-none-any.whl, ...other wheels}`
/// * `built-wheels-v0/git/<digest(url)>/<git sha>/foo/foo-1.0.0.zip/{metadata.msgpack, foo-1.0.0-py3-none-any.whl, ...other wheels}` /// * `built-wheels-v0/git/<digest(url)>/<git sha>/foo/foo-1.0.0.zip/{metadata.msgpack, foo-1.0.0-py3-none-any.whl, ...other wheels}`
/// ///
/// But the url filename does not need to be a valid source dist filename /// But the url filename does not need to be a valid source dist filename
@ -322,34 +322,27 @@ pub enum CacheBucket {
/// ├── git /// ├── git
/// │ └── a67db8ed076e3814 /// │ └── a67db8ed076e3814
/// │ └── 843b753e9e8cb74e83cac55598719b39a4d5ef1f /// │ └── 843b753e9e8cb74e83cac55598719b39a4d5ef1f
/// │ ├── manifest.msgpack
/// │ ├── metadata.msgpack /// │ ├── metadata.msgpack
/// │ └── pydantic_extra_types-2.1.0-py3-none-any.whl /// │ └── pydantic_extra_types-2.1.0-py3-none-any.whl
/// ├── pypi /// ├── pypi
/// │ └── django /// │ └── django
/// │ └── django-allauth-0.51.0.tar.gz /// │ └── django-allauth-0.51.0.tar.gz
/// │ ├── django_allauth-0.51.0-py3-none-any.whl /// │ ├── django_allauth-0.51.0-py3-none-any.whl
/// │ ├── manifest.msgpack
/// │ └── metadata.msgpack /// │ └── metadata.msgpack
/// └── url /// └── url
/// └── 6781bd6440ae72c2 /// └── 6781bd6440ae72c2
/// └── werkzeug /// └── werkzeug
/// └── werkzeug-3.0.1.tar.gz /// └── werkzeug-3.0.1.tar.gz
/// ├── manifest.msgpack
/// ├── metadata.msgpack /// ├── metadata.msgpack
/// └── werkzeug-3.0.1-py3-none-any.whl /// └── werkzeug-3.0.1-py3-none-any.whl
/// ``` /// ```
/// ///
/// Structurally, the inside of a `metadata.msgpack` looks like: /// Structurally, the `manifest.msgpack` is empty, and only contains the caching information
/// ```json /// needed to invalidate the cache. The `metadata.msgpack` contains the metadata of the source
/// { /// distribution.
/// "data": {
/// "django_allauth-0.51.0-py3-none-any.whl": {
/// "metadata-version": "2.1",
/// "name": "django-allauth",
/// "version": "0.51.0",
/// ...
/// }
/// }
/// }
/// ```
BuiltWheels, BuiltWheels,
/// Flat index responses, a format very similar to the simple metadata API. /// Flat index responses, a format very similar to the simple metadata API.
/// ///

51
crates/puffin-distribution/src/source/built_wheel_metadata.rs
View File

@ -1,12 +1,10 @@
use std::path::PathBuf; use std::path::PathBuf;
use std::str::FromStr;
use tracing::warn;
use distribution_filename::WheelFilename; use distribution_filename::WheelFilename;
use platform_tags::Tags; use platform_tags::Tags;
use puffin_cache::CacheEntry; use puffin_cache::CacheEntry;
use puffin_fs::directories;
use crate::source::manifest::{DiskFilenameAndMetadata, Manifest};
/// The information about the wheel we either just built or got from the cache. /// The information about the wheel we either just built or got from the cache.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@ -21,37 +19,26 @@ pub struct BuiltWheelMetadata {
impl BuiltWheelMetadata { impl BuiltWheelMetadata {
/// Find a compatible wheel in the cache based on the given manifest. /// Find a compatible wheel in the cache based on the given manifest.
pub(crate) fn find_in_cache( pub(crate) fn find_in_cache(tags: &Tags, cache_entry: &CacheEntry) -> Option<Self> {
tags: &Tags, for directory in directories(cache_entry.dir()) {
manifest: &Manifest, if let Some(metadata) = Self::from_path(directory) {
cache_entry: &CacheEntry, // Validate that the wheel is compatible with the target platform.
) -> Option<Self> { if metadata.filename.is_compatible(tags) {
// Find a compatible cache entry in the manifest. return Some(metadata);
let (filename, wheel) = manifest.find_wheel(tags)?; }
let metadata = Self::from_cached(filename.clone(), wheel.clone(), cache_entry); }
// Validate that the wheel exists on disk.
if !metadata.path.is_file() {
warn!(
"Wheel `{}` is present in the manifest, but not on disk",
metadata.path.display()
);
return None;
} }
None
Some(metadata)
} }
/// Create a [`BuiltWheelMetadata`] from a cached entry. /// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any.whl`).
pub(crate) fn from_cached( fn from_path(path: PathBuf) -> Option<Self> {
filename: WheelFilename, let filename = path.file_name()?.to_str()?;
cached_dist: DiskFilenameAndMetadata, let filename = WheelFilename::from_str(filename).ok()?;
cache_entry: &CacheEntry, Some(Self {
) -> Self { target: path.join(filename.stem()),
Self { path,
path: cache_entry.dir().join(cached_dist.disk_filename),
target: cache_entry.dir().join(filename.stem()),
filename, filename,
} })
} }
} }

63
crates/puffin-distribution/src/source/manifest.rs
View File

@ -1,62 +1,7 @@
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use distribution_filename::WheelFilename; /// The [`Manifest`] exists as an empty serializable struct we can use to test for cache freshness.
use platform_tags::Tags; ///
use pypi_types::Metadata21; /// TODO(charlie): Store a unique ID, rather than an empty struct.
#[derive(Debug, Default, Clone, Serialize, Deserialize)] #[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub(crate) struct Manifest { pub(crate) struct Manifest;
/// The metadata for the distribution, as returned by `prepare_metadata_for_build_wheel`.
metadata: Option<Metadata21>,
/// The built wheels for the distribution, each of which was returned from `build_wheel`.
built_wheels: FxHashMap<WheelFilename, DiskFilenameAndMetadata>,
}
impl Manifest {
/// Set the prepared metadata.
pub(crate) fn set_metadata(&mut self, metadata: Metadata21) {
self.metadata = Some(metadata);
}
/// Insert a built wheel into the manifest.
pub(crate) fn insert_wheel(
&mut self,
filename: WheelFilename,
disk_filename_and_metadata: DiskFilenameAndMetadata,
) {
self.built_wheels
.insert(filename, disk_filename_and_metadata);
}
/// Find a compatible wheel in the manifest.
pub(crate) fn find_wheel(
&self,
tags: &Tags,
) -> Option<(&WheelFilename, &DiskFilenameAndMetadata)> {
self.built_wheels
.iter()
.find(|(filename, _)| filename.is_compatible(tags))
}
/// Find a metadata in the manifest.
pub(crate) fn find_metadata(&self) -> Option<&Metadata21> {
// If we already have a prepared metadata, return it.
if let Some(metadata) = &self.metadata {
return Some(metadata);
}
// Otherwise, return the metadata from any of the built wheels.
let wheel = self.built_wheels.values().next()?;
Some(&wheel.metadata)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct DiskFilenameAndMetadata {
/// Relative, un-normalized wheel filename in the cache, which can be different than
/// `WheelFilename::to_string`.
pub(crate) disk_filename: String,
/// The [`Metadata21`] of the wheel.
pub(crate) metadata: Metadata21,
}

286
crates/puffin-distribution/src/source/mod.rs
View File

@ -23,7 +23,7 @@ use install_wheel_rs::read_dist_info;
use pep508_rs::VerbatimUrl; use pep508_rs::VerbatimUrl;
use platform_tags::Tags; use platform_tags::Tags;
use puffin_cache::{CacheBucket, CacheEntry, CacheShard, CachedByTimestamp, WheelCache}; use puffin_cache::{CacheBucket, CacheEntry, CacheShard, CachedByTimestamp, WheelCache};
use puffin_client::{CachedClient, CachedClientError, DataWithCachePolicy}; use puffin_client::{CachedClient, CachedClientError};
use puffin_fs::{write_atomic, LockedFile}; use puffin_fs::{write_atomic, LockedFile};
use puffin_git::{Fetch, GitSource}; use puffin_git::{Fetch, GitSource};
use puffin_traits::{BuildContext, BuildKind, SourceBuildTrait}; use puffin_traits::{BuildContext, BuildKind, SourceBuildTrait};
@ -32,7 +32,7 @@ use pypi_types::Metadata21;
use crate::reporter::Facade; use crate::reporter::Facade;
use crate::source::built_wheel_metadata::BuiltWheelMetadata; use crate::source::built_wheel_metadata::BuiltWheelMetadata;
pub use crate::source::error::SourceDistError; pub use crate::source::error::SourceDistError;
use crate::source::manifest::{DiskFilenameAndMetadata, Manifest}; use crate::source::manifest::Manifest;
use crate::Reporter; use crate::Reporter;
mod built_wheel_metadata; mod built_wheel_metadata;
@ -50,6 +50,9 @@ pub struct SourceDistCachedBuilder<'a, T: BuildContext> {
/// The name of the file that contains the cached manifest, encoded via `MsgPack`. /// The name of the file that contains the cached manifest, encoded via `MsgPack`.
const MANIFEST: &str = "manifest.msgpack"; const MANIFEST: &str = "manifest.msgpack";
/// The name of the file that contains the cached distribution metadata, encoded via `MsgPack`.
const METADATA: &str = "metadata.msgpack";
impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> { impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
/// Initialize a [`SourceDistCachedBuilder`] from a [`BuildContext`]. /// Initialize a [`SourceDistCachedBuilder`] from a [`BuildContext`].
pub fn new(build_context: &'a T, cached_client: &'a CachedClient, tags: &'a Tags) -> Self { pub fn new(build_context: &'a T, cached_client: &'a CachedClient, tags: &'a Tags) -> Self {
@ -262,13 +265,12 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
self.persist_source_dist_url(response, source_dist, filename, &source_dist_entry) self.persist_source_dist_url(response, source_dist, filename, &source_dist_entry)
.await?; .await?;
Ok(Manifest::default()) Ok(Manifest)
} }
.instrument(info_span!("download", source_dist = %source_dist)) .instrument(info_span!("download", source_dist = %source_dist))
}; };
let req = self.cached_client.uncached().get(url.clone()).build()?; let req = self.cached_client.uncached().get(url.clone()).build()?;
let manifest = self self.cached_client
.cached_client
.get_cached_with_callback(req, &cache_entry, download) .get_cached_with_callback(req, &cache_entry, download)
.await .await
.map_err(|err| match err { .map_err(|err| match err {
@ -277,9 +279,7 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
})?; })?;
// If the cache contains a compatible wheel, return it. // If the cache contains a compatible wheel, return it.
if let Some(built_wheel) = if let Some(built_wheel) = BuiltWheelMetadata::find_in_cache(self.tags, &cache_entry) {
BuiltWheelMetadata::find_in_cache(self.tags, &manifest, &cache_entry)
{
return Ok(built_wheel); return Ok(built_wheel);
} }
@ -301,37 +301,24 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
) )
.await?; .await?;
let cached_data = DiskFilenameAndMetadata {
disk_filename: disk_filename.clone(),
metadata: metadata.clone(),
};
// Not elegant that we have to read again here, but also not too relevant given that we
// have to build a source dist next.
// Just return if the response wasn't cacheable or there was another errors that
// `CachedClient` already complained about
if let Ok(cached) = fs::read(cache_entry.path()).await {
// If the file exists and it was just read or written by `CachedClient`, we assume it
// must be correct.
let mut cached = rmp_serde::from_slice::<DataWithCachePolicy<Manifest>>(&cached)?;
cached
.data
.insert_wheel(wheel_filename.clone(), cached_data.clone());
write_atomic(cache_entry.path(), rmp_serde::to_vec(&cached)?).await?;
};
if let Some(task) = task { if let Some(task) = task {
if let Some(reporter) = self.reporter.as_ref() { if let Some(reporter) = self.reporter.as_ref() {
reporter.on_build_complete(source_dist, task); reporter.on_build_complete(source_dist, task);
} }
} }
Ok(BuiltWheelMetadata::from_cached( // Store the metadata.
wheel_filename, let cache_entry = cache_entry.with_file(METADATA);
cached_data, write_atomic(cache_entry.path(), rmp_serde::to_vec(&metadata)?).await?;
&cache_entry,
)) let path = cache_entry.dir().join(&disk_filename);
let target = cache_entry.dir().join(wheel_filename.stem());
Ok(BuiltWheelMetadata {
path,
target,
filename: wheel_filename,
})
} }
/// Build the source distribution's metadata from a local path. /// Build the source distribution's metadata from a local path.
@ -366,13 +353,12 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
self.persist_source_dist_url(response, source_dist, filename, &source_dist_entry) self.persist_source_dist_url(response, source_dist, filename, &source_dist_entry)
.await?; .await?;
Ok(Manifest::default()) Ok(Manifest)
} }
.instrument(info_span!("download", source_dist = %source_dist)) .instrument(info_span!("download", source_dist = %source_dist))
}; };
let req = self.cached_client.uncached().get(url.clone()).build()?; let req = self.cached_client.uncached().get(url.clone()).build()?;
let manifest = self self.cached_client
.cached_client
.get_cached_with_callback(req, &cache_entry, download) .get_cached_with_callback(req, &cache_entry, download)
.await .await
.map_err(|err| match err { .map_err(|err| match err {
@ -381,7 +367,7 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
})?; })?;
// If the cache contains compatible metadata, return it. // If the cache contains compatible metadata, return it.
if let Some(metadata) = manifest.find_metadata() { if let Some(metadata) = Self::read_metadata(&cache_entry.with_file(METADATA)).await? {
return Ok(metadata.clone()); return Ok(metadata.clone());
} }
@ -394,12 +380,10 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
.boxed() .boxed()
.await? .await?
{ {
if let Ok(cached) = fs::read(cache_entry.path()).await { // Store the metadata.
let mut cached = rmp_serde::from_slice::<DataWithCachePolicy<Manifest>>(&cached)?; let cache_entry = cache_entry.with_file(METADATA);
fs::create_dir_all(cache_entry.dir()).await?;
cached.data.set_metadata(metadata.clone()); write_atomic(cache_entry.path(), rmp_serde::to_vec(&metadata)?).await?;
write_atomic(cache_entry.path(), rmp_serde::to_vec(&cached)?).await?;
};
return Ok(metadata); return Ok(metadata);
} }
@ -412,7 +396,7 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
.map(|reporter| reporter.on_build_start(source_dist)); .map(|reporter| reporter.on_build_start(source_dist));
// Build the source distribution. // Build the source distribution.
let (disk_filename, wheel_filename, metadata) = self let (_disk_filename, _wheel_filename, metadata) = self
.build_source_dist( .build_source_dist(
source_dist, source_dist,
source_dist_entry.path(), source_dist_entry.path(),
@ -421,24 +405,9 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
) )
.await?; .await?;
// Not elegant that we have to read again here, but also not too relevant given that we // Store the metadata.
// have to build a source dist next. let cache_entry = cache_entry.with_file(METADATA);
// Just return if the response wasn't cacheable or there was another errors that write_atomic(cache_entry.path(), rmp_serde::to_vec(&metadata)?).await?;
// `CachedClient` already complained about
if let Ok(cached) = fs::read(cache_entry.path()).await {
// If the file exists and it was just read or written by `CachedClient`, we assume it
// must be correct.
let mut cached = rmp_serde::from_slice::<DataWithCachePolicy<Manifest>>(&cached)?;
cached.data.insert_wheel(
wheel_filename.clone(),
DiskFilenameAndMetadata {
disk_filename: disk_filename.clone(),
metadata: metadata.clone(),
},
);
write_atomic(cache_entry.path(), rmp_serde::to_vec(&cached)?).await?;
};
if let Some(task) = task { if let Some(task) = task {
if let Some(reporter) = self.reporter.as_ref() { if let Some(reporter) = self.reporter.as_ref() {
@ -467,15 +436,11 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
return Err(SourceDistError::DirWithoutEntrypoint); return Err(SourceDistError::DirWithoutEntrypoint);
}; };
// Read the existing metadata from the cache. // Read the existing metadata from the cache, to clear stale wheels.
let mut manifest = Self::read_cached_manifest(&cache_entry, modified) Self::refresh_cached_manifest(&cache_entry, modified).await?;
.await?
.unwrap_or_default();
// If the cache contains a compatible wheel, return it. // If the cache contains a compatible wheel, return it.
if let Some(built_wheel) = if let Some(built_wheel) = BuiltWheelMetadata::find_in_cache(self.tags, &cache_entry) {
BuiltWheelMetadata::find_in_cache(self.tags, &manifest, &cache_entry)
{
return Ok(built_wheel); return Ok(built_wheel);
} }
@ -489,27 +454,16 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
.build_source_dist(source_dist, &path_source_dist.path, None, &cache_entry) .build_source_dist(source_dist, &path_source_dist.path, None, &cache_entry)
.await?; .await?;
// Store the metadata for this build along with all the other builds.
manifest.insert_wheel(
filename.clone(),
DiskFilenameAndMetadata {
disk_filename: disk_filename.clone(),
metadata: metadata.clone(),
},
);
let cached = CachedByTimestamp {
timestamp: modified,
data: manifest,
};
let data = rmp_serde::to_vec(&cached)?;
write_atomic(cache_entry.path(), data).await?;
if let Some(task) = task { if let Some(task) = task {
if let Some(reporter) = self.reporter.as_ref() { if let Some(reporter) = self.reporter.as_ref() {
reporter.on_build_complete(source_dist, task); reporter.on_build_complete(source_dist, task);
} }
} }
// Store the metadata.
let cache_entry = cache_entry.with_file(METADATA);
write_atomic(cache_entry.path(), rmp_serde::to_vec(&metadata)?).await?;
let path = cache_entry.dir().join(&disk_filename); let path = cache_entry.dir().join(&disk_filename);
let target = cache_entry.dir().join(filename.stem()); let target = cache_entry.dir().join(filename.stem());
@ -541,13 +495,11 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
return Err(SourceDistError::DirWithoutEntrypoint); return Err(SourceDistError::DirWithoutEntrypoint);
}; };
// Read the existing metadata from the cache. // Read the existing metadata from the cache, to clear stale entries.
let mut manifest = Self::read_cached_manifest(&cache_entry, modified) Self::refresh_cached_manifest(&cache_entry, modified).await?;
.await?
.unwrap_or_default();
// If the cache contains compatible metadata, return it. // If the cache contains compatible metadata, return it.
if let Some(metadata) = manifest.find_metadata() { if let Some(metadata) = Self::read_metadata(&cache_entry.with_file(METADATA)).await? {
return Ok(metadata.clone()); return Ok(metadata.clone());
} }
@ -557,16 +509,9 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
.boxed() .boxed()
.await? .await?
{ {
// Store the metadata for this build along with all the other builds. // Store the metadata.
manifest.set_metadata(metadata.clone()); let cache_entry = cache_entry.with_file(METADATA);
let cached = CachedByTimestamp { write_atomic(cache_entry.path(), rmp_serde::to_vec(&metadata)?).await?;
timestamp: modified,
data: manifest,
};
let data = rmp_serde::to_vec(&cached)?;
fs::create_dir_all(&cache_entry.dir()).await?;
write_atomic(cache_entry.path(), data).await?;
return Ok(metadata); return Ok(metadata);
} }
@ -577,31 +522,20 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
.as_ref() .as_ref()
.map(|reporter| reporter.on_build_start(source_dist)); .map(|reporter| reporter.on_build_start(source_dist));
let (disk_filename, filename, metadata) = self let (_disk_filename, _filename, metadata) = self
.build_source_dist(source_dist, &path_source_dist.path, None, &cache_entry) .build_source_dist(source_dist, &path_source_dist.path, None, &cache_entry)
.await?; .await?;
// Store the metadata for this build along with all the other builds.
manifest.insert_wheel(
filename.clone(),
DiskFilenameAndMetadata {
disk_filename: disk_filename.clone(),
metadata: metadata.clone(),
},
);
let cached = CachedByTimestamp {
timestamp: modified,
data: manifest,
};
let data = rmp_serde::to_vec(&cached)?;
write_atomic(cache_entry.path(), data).await?;
if let Some(task) = task { if let Some(task) = task {
if let Some(reporter) = self.reporter.as_ref() { if let Some(reporter) = self.reporter.as_ref() {
reporter.on_build_complete(source_dist, task); reporter.on_build_complete(source_dist, task);
} }
} }
// Store the metadata.
let cache_entry = cache_entry.with_file(METADATA);
write_atomic(cache_entry.path(), rmp_serde::to_vec(&metadata)?).await?;
Ok(metadata) Ok(metadata)
} }
@ -613,7 +547,6 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
) -> Result<BuiltWheelMetadata, SourceDistError> { ) -> Result<BuiltWheelMetadata, SourceDistError> {
let (fetch, subdirectory) = self.download_source_dist_git(&git_source_dist.url).await?; let (fetch, subdirectory) = self.download_source_dist_git(&git_source_dist.url).await?;
// TODO(konstin): Do we want to delete old built wheels when the git sha changed?
let git_sha = fetch.git().precise().expect("Exact commit after checkout"); let git_sha = fetch.git().precise().expect("Exact commit after checkout");
let cache_entry = self.build_context.cache().entry( let cache_entry = self.build_context.cache().entry(
CacheBucket::BuiltWheels, CacheBucket::BuiltWheels,
@ -622,13 +555,11 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
MANIFEST, MANIFEST,
); );
// Read the existing metadata from the cache. // Read the existing metadata from the cache, to clear stale entries.
let mut manifest = Self::read_manifest(&cache_entry).await?.unwrap_or_default(); Self::refresh_manifest(&cache_entry).await?;
// If the cache contains a compatible wheel, return it. // If the cache contains a compatible wheel, return it.
if let Some(built_wheel) = if let Some(built_wheel) = BuiltWheelMetadata::find_in_cache(self.tags, &cache_entry) {
BuiltWheelMetadata::find_in_cache(self.tags, &manifest, &cache_entry)
{
return Ok(built_wheel); return Ok(built_wheel);
} }
@ -646,23 +577,16 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
) )
.await?; .await?;
// Store the metadata for this build along with all the other builds.
manifest.insert_wheel(
filename.clone(),
DiskFilenameAndMetadata {
disk_filename: disk_filename.clone(),
metadata: metadata.clone(),
},
);
let data = rmp_serde::to_vec(&manifest)?;
write_atomic(cache_entry.path(), data).await?;
if let Some(task) = task { if let Some(task) = task {
if let Some(reporter) = self.reporter.as_ref() { if let Some(reporter) = self.reporter.as_ref() {
reporter.on_build_complete(source_dist, task); reporter.on_build_complete(source_dist, task);
} }
} }
// Store the metadata.
let cache_entry = cache_entry.with_file(METADATA);
write_atomic(cache_entry.path(), rmp_serde::to_vec(&metadata)?).await?;
let path = cache_entry.dir().join(&disk_filename); let path = cache_entry.dir().join(&disk_filename);
let target = cache_entry.dir().join(filename.stem()); let target = cache_entry.dir().join(filename.stem());
@ -692,11 +616,11 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
MANIFEST, MANIFEST,
); );
// Read the existing metadata from the cache. // Read the existing metadata from the cache, to clear stale entries.
let mut manifest = Self::read_manifest(&cache_entry).await?.unwrap_or_default(); Self::refresh_manifest(&cache_entry).await?;
// If the cache contains compatible metadata, return it. // If the cache contains compatible metadata, return it.
if let Some(metadata) = manifest.find_metadata() { if let Some(metadata) = Self::read_metadata(&cache_entry.with_file(METADATA)).await? {
return Ok(metadata.clone()); return Ok(metadata.clone());
} }
@ -706,12 +630,9 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
.boxed() .boxed()
.await? .await?
{ {
// Store the metadata for this build along with all the other builds. // Store the metadata.
manifest.set_metadata(metadata.clone()); let cache_entry = cache_entry.with_file(METADATA);
let data = rmp_serde::to_vec(&manifest)?; write_atomic(cache_entry.path(), rmp_serde::to_vec(&metadata)?).await?;
fs::create_dir_all(&cache_entry.dir()).await?;
write_atomic(cache_entry.path(), data).await?;
return Ok(metadata); return Ok(metadata);
} }
@ -722,7 +643,7 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
.as_ref() .as_ref()
.map(|reporter| reporter.on_build_start(source_dist)); .map(|reporter| reporter.on_build_start(source_dist));
let (disk_filename, filename, metadata) = self let (_disk_filename, _filename, metadata) = self
.build_source_dist( .build_source_dist(
source_dist, source_dist,
fetch.path(), fetch.path(),
@ -731,23 +652,16 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
) )
.await?; .await?;
// Store the metadata for this build along with all the other builds.
manifest.insert_wheel(
filename.clone(),
DiskFilenameAndMetadata {
disk_filename: disk_filename.clone(),
metadata: metadata.clone(),
},
);
let data = rmp_serde::to_vec(&manifest)?;
write_atomic(cache_entry.path(), data).await?;
if let Some(task) = task { if let Some(task) = task {
if let Some(reporter) = self.reporter.as_ref() { if let Some(reporter) = self.reporter.as_ref() {
reporter.on_build_complete(source_dist, task); reporter.on_build_complete(source_dist, task);
} }
} }
// Store the metadata.
let cache_entry = cache_entry.with_file(METADATA);
write_atomic(cache_entry.path(), rmp_serde::to_vec(&metadata)?).await?;
Ok(metadata) Ok(metadata)
} }
@ -975,7 +889,7 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
.await .await
.map_err(|err| SourceDistError::BuildEditable(editable.to_string(), err))?; .map_err(|err| SourceDistError::BuildEditable(editable.to_string(), err))?;
let filename = WheelFilename::from_str(&disk_filename)?; let filename = WheelFilename::from_str(&disk_filename)?;
// We finally have the name of the package and can construct the dist // We finally have the name of the package and can construct the dist.
let dist = Dist::Source(SourceDist::Path(PathSourceDist { let dist = Dist::Source(SourceDist::Path(PathSourceDist {
name: filename.name.clone(), name: filename.name.clone(),
url: editable.url().clone(), url: editable.url().clone(),
@ -989,35 +903,67 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
} }
/// Read an existing cached [`Manifest`], if it exists and is up-to-date. /// Read an existing cached [`Manifest`], if it exists and is up-to-date.
async fn read_cached_manifest( ///
/// If the cache entry is stale, it will be removed and recreated.
async fn refresh_cached_manifest(
cache_entry: &CacheEntry, cache_entry: &CacheEntry,
modified: std::time::SystemTime, modified: std::time::SystemTime,
) -> Result<Option<Manifest>, SourceDistError> { ) -> Result<Manifest, SourceDistError> {
// If the cache entry is up-to-date, return it; if it's stale, remove it.
match fs::read(&cache_entry.path()).await { match fs::read(&cache_entry.path()).await {
Ok(cached) => { Ok(cached) => {
let cached = rmp_serde::from_slice::<CachedByTimestamp<Manifest>>(&cached)?; let cached = rmp_serde::from_slice::<CachedByTimestamp<Manifest>>(&cached)?;
if cached.timestamp == modified { if cached.timestamp == modified {
Ok(Some(cached.data)) return Ok(cached.data);
} else { }
debug!(
"Removing stale built wheels for: {}", debug!(
cache_entry.path().display() "Removing stale built wheels for: {}",
); cache_entry.path().display()
if let Err(err) = fs::remove_dir_all(&cache_entry.dir()).await { );
warn!("Failed to remove stale built wheel cache directory: {err}"); if let Err(err) = fs::remove_dir_all(&cache_entry.dir()).await {
} warn!("Failed to remove stale built wheel cache directory: {err}");
Ok(None)
} }
} }
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
Err(err) => return Err(err.into()),
}
// Write a new cache entry.
fs::create_dir_all(&cache_entry.dir()).await?;
write_atomic(
cache_entry.path(),
rmp_serde::to_vec(&CachedByTimestamp {
timestamp: modified,
data: Manifest,
})?,
)
.await?;
Ok(Manifest)
}
/// Read an existing cached [`Manifest`], if it exists.
///
/// If the cache entry does not exist, it will be created.
async fn refresh_manifest(cache_entry: &CacheEntry) -> Result<Manifest, SourceDistError> {
match fs::read(&cache_entry.path()).await {
Ok(cached) => Ok(rmp_serde::from_slice::<Manifest>(&cached)?),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
fs::create_dir_all(&cache_entry.dir()).await?;
write_atomic(cache_entry.path(), rmp_serde::to_vec(&Manifest)?).await?;
Ok(Manifest)
}
Err(err) => Err(err.into()), Err(err) => Err(err.into()),
} }
} }
/// Read an existing cached [`Manifest`], if it exists. /// Read an existing cached [`Metadata21`], if it exists.
async fn read_manifest(cache_entry: &CacheEntry) -> Result<Option<Manifest>, SourceDistError> { async fn read_metadata(
cache_entry: &CacheEntry,
) -> Result<Option<Metadata21>, SourceDistError> {
match fs::read(&cache_entry.path()).await { match fs::read(&cache_entry.path()).await {
Ok(cached) => Ok(Some(rmp_serde::from_slice::<Manifest>(&cached)?)), Ok(cached) => Ok(Some(rmp_serde::from_slice::<Metadata21>(&cached)?)),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(err) => Err(err.into()), Err(err) => Err(err.into()),
} }