mirror of https://github.com/astral-sh/uv
Cache Python downloads by default
i.e., without requiring opt-in vi `UV_PYTHON_CACHE_DIR`
This commit is contained in:
parent
caac4814df
commit
6d69bda50b
|
|
@ -21,6 +21,7 @@ use tokio_util::either::Either;
|
|||
use tracing::{debug, instrument};
|
||||
use url::Url;
|
||||
|
||||
use uv_cache::{Cache, CacheBucket};
|
||||
use uv_client::{BaseClient, WrappedReqwestError, is_transient_network_error};
|
||||
use uv_distribution_filename::{ExtensionError, SourceDistExtension};
|
||||
use uv_extract::hash::Hasher;
|
||||
|
|
@ -1099,13 +1100,14 @@ impl ManagedPythonDownload {
|
|||
}
|
||||
|
||||
/// Download and extract a Python distribution, retrying on failure.
|
||||
#[instrument(skip(client, installation_dir, scratch_dir, reporter), fields(download = % self.key()))]
|
||||
#[instrument(skip(client, installation_dir, scratch_dir, cache, reporter), fields(download = % self.key()))]
|
||||
pub async fn fetch_with_retry(
|
||||
&self,
|
||||
client: &BaseClient,
|
||||
retry_policy: &ExponentialBackoff,
|
||||
installation_dir: &Path,
|
||||
scratch_dir: &Path,
|
||||
cache: &Cache,
|
||||
reinstall: bool,
|
||||
python_install_mirror: Option<&str>,
|
||||
pypy_install_mirror: Option<&str>,
|
||||
|
|
@ -1120,6 +1122,7 @@ impl ManagedPythonDownload {
|
|||
client,
|
||||
installation_dir,
|
||||
scratch_dir,
|
||||
cache,
|
||||
reinstall,
|
||||
python_install_mirror,
|
||||
pypy_install_mirror,
|
||||
|
|
@ -1167,12 +1170,13 @@ impl ManagedPythonDownload {
|
|||
}
|
||||
|
||||
/// Download and extract a Python distribution.
|
||||
#[instrument(skip(client, installation_dir, scratch_dir, reporter), fields(download = % self.key()))]
|
||||
#[instrument(skip(client, installation_dir, scratch_dir, cache, reporter), fields(download = % self.key()))]
|
||||
pub async fn fetch(
|
||||
&self,
|
||||
client: &BaseClient,
|
||||
installation_dir: &Path,
|
||||
scratch_dir: &Path,
|
||||
cache: &Cache,
|
||||
reinstall: bool,
|
||||
python_install_mirror: Option<&str>,
|
||||
pypy_install_mirror: Option<&str>,
|
||||
|
|
@ -1205,93 +1209,76 @@ impl ManagedPythonDownload {
|
|||
|
||||
let temp_dir = tempfile::tempdir_in(scratch_dir).map_err(Error::DownloadDirError)?;
|
||||
|
||||
if let Some(python_builds_dir) =
|
||||
env::var_os(EnvVars::UV_PYTHON_CACHE_DIR).filter(|s| !s.is_empty())
|
||||
{
|
||||
let python_builds_dir = PathBuf::from(python_builds_dir);
|
||||
fs_err::create_dir_all(&python_builds_dir)?;
|
||||
let hash_prefix = match self.sha256.as_deref() {
|
||||
Some(sha) => {
|
||||
// Shorten the hash to avoid too-long-filename errors
|
||||
&sha[..9]
|
||||
// Use the cache directory from the environment variable if set, otherwise use the default
|
||||
// cache bucket.
|
||||
let python_builds_dir = env::var_os(EnvVars::UV_PYTHON_CACHE_DIR)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| cache.bucket(CacheBucket::Python));
|
||||
|
||||
fs_err::create_dir_all(&python_builds_dir)?;
|
||||
let hash_prefix = match self.sha256.as_deref() {
|
||||
Some(sha) => {
|
||||
// Shorten the hash to avoid too-long-filename errors
|
||||
&sha[..9]
|
||||
}
|
||||
None => "none",
|
||||
};
|
||||
let target_cache_file = python_builds_dir.join(format!("{hash_prefix}-{filename}"));
|
||||
|
||||
// Download the archive to the cache, or return a reader if we have it in cache.
|
||||
// TODO(konsti): We should "tee" the write so we can do the download-to-cache and unpacking
|
||||
// in one step.
|
||||
let (reader, size): (Box<dyn AsyncRead + Unpin>, Option<u64>) =
|
||||
match fs_err::tokio::File::open(&target_cache_file).await {
|
||||
Ok(file) => {
|
||||
debug!(
|
||||
"Extracting existing `{}`",
|
||||
target_cache_file.simplified_display()
|
||||
);
|
||||
let size = file.metadata().await?.len();
|
||||
let reader = Box::new(tokio::io::BufReader::new(file));
|
||||
(reader, Some(size))
|
||||
}
|
||||
None => "none",
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {
|
||||
// Point the user to which file is missing where and where to download it
|
||||
if client.connectivity().is_offline() {
|
||||
return Err(Error::OfflinePythonMissing {
|
||||
file: Box::new(self.key().clone()),
|
||||
url: Box::new(url),
|
||||
python_builds_dir,
|
||||
});
|
||||
}
|
||||
|
||||
self.download_archive(
|
||||
&url,
|
||||
client,
|
||||
reporter,
|
||||
&python_builds_dir,
|
||||
&target_cache_file,
|
||||
)
|
||||
.await?;
|
||||
|
||||
debug!("Extracting `{}`", target_cache_file.simplified_display());
|
||||
let file = fs_err::tokio::File::open(&target_cache_file).await?;
|
||||
let size = file.metadata().await?.len();
|
||||
let reader = Box::new(tokio::io::BufReader::new(file));
|
||||
(reader, Some(size))
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
let target_cache_file = python_builds_dir.join(format!("{hash_prefix}-{filename}"));
|
||||
|
||||
// Download the archive to the cache, or return a reader if we have it in cache.
|
||||
// TODO(konsti): We should "tee" the write so we can do the download-to-cache and unpacking
|
||||
// in one step.
|
||||
let (reader, size): (Box<dyn AsyncRead + Unpin>, Option<u64>) =
|
||||
match fs_err::tokio::File::open(&target_cache_file).await {
|
||||
Ok(file) => {
|
||||
debug!(
|
||||
"Extracting existing `{}`",
|
||||
target_cache_file.simplified_display()
|
||||
);
|
||||
let size = file.metadata().await?.len();
|
||||
let reader = Box::new(tokio::io::BufReader::new(file));
|
||||
(reader, Some(size))
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {
|
||||
// Point the user to which file is missing where and where to download it
|
||||
if client.connectivity().is_offline() {
|
||||
return Err(Error::OfflinePythonMissing {
|
||||
file: Box::new(self.key().clone()),
|
||||
url: Box::new(url),
|
||||
python_builds_dir,
|
||||
});
|
||||
}
|
||||
|
||||
self.download_archive(
|
||||
&url,
|
||||
client,
|
||||
reporter,
|
||||
&python_builds_dir,
|
||||
&target_cache_file,
|
||||
)
|
||||
.await?;
|
||||
|
||||
debug!("Extracting `{}`", target_cache_file.simplified_display());
|
||||
let file = fs_err::tokio::File::open(&target_cache_file).await?;
|
||||
let size = file.metadata().await?.len();
|
||||
let reader = Box::new(tokio::io::BufReader::new(file));
|
||||
(reader, Some(size))
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
|
||||
// Extract the downloaded archive into a temporary directory.
|
||||
self.extract_reader(
|
||||
reader,
|
||||
temp_dir.path(),
|
||||
&filename,
|
||||
ext,
|
||||
size,
|
||||
reporter,
|
||||
Direction::Extract,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
// Avoid overlong log lines
|
||||
debug!("Downloading {url}");
|
||||
debug!(
|
||||
"Extracting {filename} to temporary location: {}",
|
||||
temp_dir.path().simplified_display()
|
||||
);
|
||||
|
||||
let (reader, size) = read_url(&url, client).await?;
|
||||
self.extract_reader(
|
||||
reader,
|
||||
temp_dir.path(),
|
||||
&filename,
|
||||
ext,
|
||||
size,
|
||||
reporter,
|
||||
Direction::Download,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
// Extract the downloaded archive into a temporary directory.
|
||||
self.extract_reader(
|
||||
reader,
|
||||
temp_dir.path(),
|
||||
&filename,
|
||||
ext,
|
||||
size,
|
||||
reporter,
|
||||
Direction::Extract,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Extract the top-level directory.
|
||||
let mut extracted = match uv_extract::strip_component(temp_dir.path()) {
|
||||
|
|
|
|||
|
|
@ -261,6 +261,7 @@ impl PythonInstallation {
|
|||
retry_policy,
|
||||
installations_dir,
|
||||
&scratch_dir,
|
||||
cache,
|
||||
false,
|
||||
python_install_mirror,
|
||||
pypy_install_mirror,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ use owo_colors::{AnsiColors, OwoColorize};
|
|||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
use tracing::{debug, trace};
|
||||
|
||||
use uv_cache::Cache;
|
||||
use uv_client::BaseClientBuilder;
|
||||
use uv_fs::Simplified;
|
||||
use uv_platform::{Arch, Libc};
|
||||
|
|
@ -188,6 +189,7 @@ pub(crate) async fn install(
|
|||
pypy_install_mirror: Option<String>,
|
||||
python_downloads_json_url: Option<String>,
|
||||
client_builder: BaseClientBuilder<'_>,
|
||||
cache: &Cache,
|
||||
default: bool,
|
||||
python_downloads: PythonDownloads,
|
||||
no_config: bool,
|
||||
|
|
@ -470,6 +472,7 @@ pub(crate) async fn install(
|
|||
&retry_policy,
|
||||
installations_dir,
|
||||
&scratch_dir,
|
||||
cache,
|
||||
reinstall,
|
||||
python_install_mirror.as_deref(),
|
||||
pypy_install_mirror.as_deref(),
|
||||
|
|
|
|||
|
|
@ -1601,6 +1601,7 @@ async fn run(mut cli: Cli) -> Result<ExitStatus> {
|
|||
let args = settings::PythonInstallSettings::resolve(args, filesystem, environment);
|
||||
show_settings!(args);
|
||||
|
||||
let cache = cache.init().await?;
|
||||
commands::python_install(
|
||||
&project_dir,
|
||||
args.install_dir,
|
||||
|
|
@ -1614,6 +1615,7 @@ async fn run(mut cli: Cli) -> Result<ExitStatus> {
|
|||
args.pypy_install_mirror,
|
||||
args.python_downloads_json_url,
|
||||
client_builder.subcommand(vec!["python".to_owned(), "install".to_owned()]),
|
||||
&cache,
|
||||
args.default,
|
||||
globals.python_downloads,
|
||||
cli.top_level.no_config,
|
||||
|
|
@ -1630,6 +1632,7 @@ async fn run(mut cli: Cli) -> Result<ExitStatus> {
|
|||
show_settings!(args);
|
||||
let upgrade = commands::PythonUpgrade::Enabled(commands::PythonUpgradeSource::Upgrade);
|
||||
|
||||
let cache = cache.init().await?;
|
||||
commands::python_install(
|
||||
&project_dir,
|
||||
args.install_dir,
|
||||
|
|
@ -1643,6 +1646,7 @@ async fn run(mut cli: Cli) -> Result<ExitStatus> {
|
|||
args.pypy_install_mirror,
|
||||
args.python_downloads_json_url,
|
||||
client_builder.subcommand(vec!["python".to_owned(), "upgrade".to_owned()]),
|
||||
&cache,
|
||||
args.default,
|
||||
globals.python_downloads,
|
||||
cli.top_level.no_config,
|
||||
|
|
|
|||
|
|
@ -2712,10 +2712,10 @@ fn python_install_no_cache() {
|
|||
- cpython-3.14.2-[PLATFORM] (python3.14)
|
||||
");
|
||||
|
||||
// 3.12 isn't cached, so it can't be installed
|
||||
// 3.12 isn't cached, so it can't be installed offline
|
||||
let mut filters = context.filters();
|
||||
filters.push((
|
||||
"cpython-3.12.*.tar.gz",
|
||||
r"cpython-3\.12\.\d+(%2B|\+)\d+-[a-z0-9_-]+\.tar\.gz",
|
||||
"cpython-3.12.[PATCH]-[DATE]-[PLATFORM].tar.gz",
|
||||
));
|
||||
filters.push((r"releases/download/\d{8}/", "releases/download/[DATE]/"));
|
||||
|
|
@ -2729,8 +2729,7 @@ fn python_install_no_cache() {
|
|||
|
||||
----- stderr -----
|
||||
error: Failed to install cpython-3.12.12-[PLATFORM]
|
||||
Caused by: Failed to download https://github.com/astral-sh/python-build-standalone/releases/download/[DATE]/cpython-3.12.[PATCH]-[DATE]-[PLATFORM].tar.gz
|
||||
Caused by: Network connectivity is disabled, but the requested data wasn't found in the cache for: `https://github.com/astral-sh/python-build-standalone/releases/download/[DATE]/cpython-3.12.[PATCH]-[DATE]-[PLATFORM].tar.gz`
|
||||
Caused by: An offline Python installation was requested, but cpython-3.12.12-[PLATFORM] (from https://github.com/astral-sh/python-build-standalone/releases/download/[DATE]/cpython-3.12.[PATCH]-[DATE]-[PLATFORM].tar.gz) is missing in [CACHE_DIR]/python-v0
|
||||
");
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue