From 6d69bda50bb518489033f4b694064d9b5afe5356 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 10 Dec 2025 10:40:15 -0600 Subject: [PATCH] Cache Python downloads by default i.e., without requiring opt-in vi `UV_PYTHON_CACHE_DIR` --- crates/uv-python/src/downloads.rs | 159 +++++++++++------------ crates/uv-python/src/installation.rs | 1 + crates/uv/src/commands/python/install.rs | 3 + crates/uv/src/lib.rs | 4 + crates/uv/tests/it/python_install.rs | 7 +- 5 files changed, 84 insertions(+), 90 deletions(-) diff --git a/crates/uv-python/src/downloads.rs b/crates/uv-python/src/downloads.rs index f500d3e1d..bd2f15807 100644 --- a/crates/uv-python/src/downloads.rs +++ b/crates/uv-python/src/downloads.rs @@ -21,6 +21,7 @@ use tokio_util::either::Either; use tracing::{debug, instrument}; use url::Url; +use uv_cache::{Cache, CacheBucket}; use uv_client::{BaseClient, WrappedReqwestError, is_transient_network_error}; use uv_distribution_filename::{ExtensionError, SourceDistExtension}; use uv_extract::hash::Hasher; @@ -1099,13 +1100,14 @@ impl ManagedPythonDownload { } /// Download and extract a Python distribution, retrying on failure. - #[instrument(skip(client, installation_dir, scratch_dir, reporter), fields(download = % self.key()))] + #[instrument(skip(client, installation_dir, scratch_dir, cache, reporter), fields(download = % self.key()))] pub async fn fetch_with_retry( &self, client: &BaseClient, retry_policy: &ExponentialBackoff, installation_dir: &Path, scratch_dir: &Path, + cache: &Cache, reinstall: bool, python_install_mirror: Option<&str>, pypy_install_mirror: Option<&str>, @@ -1120,6 +1122,7 @@ impl ManagedPythonDownload { client, installation_dir, scratch_dir, + cache, reinstall, python_install_mirror, pypy_install_mirror, @@ -1167,12 +1170,13 @@ impl ManagedPythonDownload { } /// Download and extract a Python distribution. - #[instrument(skip(client, installation_dir, scratch_dir, reporter), fields(download = % self.key()))] + #[instrument(skip(client, installation_dir, scratch_dir, cache, reporter), fields(download = % self.key()))] pub async fn fetch( &self, client: &BaseClient, installation_dir: &Path, scratch_dir: &Path, + cache: &Cache, reinstall: bool, python_install_mirror: Option<&str>, pypy_install_mirror: Option<&str>, @@ -1205,93 +1209,76 @@ impl ManagedPythonDownload { let temp_dir = tempfile::tempdir_in(scratch_dir).map_err(Error::DownloadDirError)?; - if let Some(python_builds_dir) = - env::var_os(EnvVars::UV_PYTHON_CACHE_DIR).filter(|s| !s.is_empty()) - { - let python_builds_dir = PathBuf::from(python_builds_dir); - fs_err::create_dir_all(&python_builds_dir)?; - let hash_prefix = match self.sha256.as_deref() { - Some(sha) => { - // Shorten the hash to avoid too-long-filename errors - &sha[..9] + // Use the cache directory from the environment variable if set, otherwise use the default + // cache bucket. + let python_builds_dir = env::var_os(EnvVars::UV_PYTHON_CACHE_DIR) + .filter(|s| !s.is_empty()) + .map(PathBuf::from) + .unwrap_or_else(|| cache.bucket(CacheBucket::Python)); + + fs_err::create_dir_all(&python_builds_dir)?; + let hash_prefix = match self.sha256.as_deref() { + Some(sha) => { + // Shorten the hash to avoid too-long-filename errors + &sha[..9] + } + None => "none", + }; + let target_cache_file = python_builds_dir.join(format!("{hash_prefix}-{filename}")); + + // Download the archive to the cache, or return a reader if we have it in cache. + // TODO(konsti): We should "tee" the write so we can do the download-to-cache and unpacking + // in one step. + let (reader, size): (Box, Option) = + match fs_err::tokio::File::open(&target_cache_file).await { + Ok(file) => { + debug!( + "Extracting existing `{}`", + target_cache_file.simplified_display() + ); + let size = file.metadata().await?.len(); + let reader = Box::new(tokio::io::BufReader::new(file)); + (reader, Some(size)) } - None => "none", + Err(err) if err.kind() == io::ErrorKind::NotFound => { + // Point the user to which file is missing where and where to download it + if client.connectivity().is_offline() { + return Err(Error::OfflinePythonMissing { + file: Box::new(self.key().clone()), + url: Box::new(url), + python_builds_dir, + }); + } + + self.download_archive( + &url, + client, + reporter, + &python_builds_dir, + &target_cache_file, + ) + .await?; + + debug!("Extracting `{}`", target_cache_file.simplified_display()); + let file = fs_err::tokio::File::open(&target_cache_file).await?; + let size = file.metadata().await?.len(); + let reader = Box::new(tokio::io::BufReader::new(file)); + (reader, Some(size)) + } + Err(err) => return Err(err.into()), }; - let target_cache_file = python_builds_dir.join(format!("{hash_prefix}-{filename}")); - // Download the archive to the cache, or return a reader if we have it in cache. - // TODO(konsti): We should "tee" the write so we can do the download-to-cache and unpacking - // in one step. - let (reader, size): (Box, Option) = - match fs_err::tokio::File::open(&target_cache_file).await { - Ok(file) => { - debug!( - "Extracting existing `{}`", - target_cache_file.simplified_display() - ); - let size = file.metadata().await?.len(); - let reader = Box::new(tokio::io::BufReader::new(file)); - (reader, Some(size)) - } - Err(err) if err.kind() == io::ErrorKind::NotFound => { - // Point the user to which file is missing where and where to download it - if client.connectivity().is_offline() { - return Err(Error::OfflinePythonMissing { - file: Box::new(self.key().clone()), - url: Box::new(url), - python_builds_dir, - }); - } - - self.download_archive( - &url, - client, - reporter, - &python_builds_dir, - &target_cache_file, - ) - .await?; - - debug!("Extracting `{}`", target_cache_file.simplified_display()); - let file = fs_err::tokio::File::open(&target_cache_file).await?; - let size = file.metadata().await?.len(); - let reader = Box::new(tokio::io::BufReader::new(file)); - (reader, Some(size)) - } - Err(err) => return Err(err.into()), - }; - - // Extract the downloaded archive into a temporary directory. - self.extract_reader( - reader, - temp_dir.path(), - &filename, - ext, - size, - reporter, - Direction::Extract, - ) - .await?; - } else { - // Avoid overlong log lines - debug!("Downloading {url}"); - debug!( - "Extracting {filename} to temporary location: {}", - temp_dir.path().simplified_display() - ); - - let (reader, size) = read_url(&url, client).await?; - self.extract_reader( - reader, - temp_dir.path(), - &filename, - ext, - size, - reporter, - Direction::Download, - ) - .await?; - } + // Extract the downloaded archive into a temporary directory. + self.extract_reader( + reader, + temp_dir.path(), + &filename, + ext, + size, + reporter, + Direction::Extract, + ) + .await?; // Extract the top-level directory. let mut extracted = match uv_extract::strip_component(temp_dir.path()) { diff --git a/crates/uv-python/src/installation.rs b/crates/uv-python/src/installation.rs index 2cc751bda..059aa1510 100644 --- a/crates/uv-python/src/installation.rs +++ b/crates/uv-python/src/installation.rs @@ -261,6 +261,7 @@ impl PythonInstallation { retry_policy, installations_dir, &scratch_dir, + cache, false, python_install_mirror, pypy_install_mirror, diff --git a/crates/uv/src/commands/python/install.rs b/crates/uv/src/commands/python/install.rs index 480731cbe..3eeab220c 100644 --- a/crates/uv/src/commands/python/install.rs +++ b/crates/uv/src/commands/python/install.rs @@ -14,6 +14,7 @@ use owo_colors::{AnsiColors, OwoColorize}; use rustc_hash::{FxHashMap, FxHashSet}; use tracing::{debug, trace}; +use uv_cache::Cache; use uv_client::BaseClientBuilder; use uv_fs::Simplified; use uv_platform::{Arch, Libc}; @@ -188,6 +189,7 @@ pub(crate) async fn install( pypy_install_mirror: Option, python_downloads_json_url: Option, client_builder: BaseClientBuilder<'_>, + cache: &Cache, default: bool, python_downloads: PythonDownloads, no_config: bool, @@ -470,6 +472,7 @@ pub(crate) async fn install( &retry_policy, installations_dir, &scratch_dir, + cache, reinstall, python_install_mirror.as_deref(), pypy_install_mirror.as_deref(), diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index d8d4dbb91..41f95a074 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -1601,6 +1601,7 @@ async fn run(mut cli: Cli) -> Result { let args = settings::PythonInstallSettings::resolve(args, filesystem, environment); show_settings!(args); + let cache = cache.init().await?; commands::python_install( &project_dir, args.install_dir, @@ -1614,6 +1615,7 @@ async fn run(mut cli: Cli) -> Result { args.pypy_install_mirror, args.python_downloads_json_url, client_builder.subcommand(vec!["python".to_owned(), "install".to_owned()]), + &cache, args.default, globals.python_downloads, cli.top_level.no_config, @@ -1630,6 +1632,7 @@ async fn run(mut cli: Cli) -> Result { show_settings!(args); let upgrade = commands::PythonUpgrade::Enabled(commands::PythonUpgradeSource::Upgrade); + let cache = cache.init().await?; commands::python_install( &project_dir, args.install_dir, @@ -1643,6 +1646,7 @@ async fn run(mut cli: Cli) -> Result { args.pypy_install_mirror, args.python_downloads_json_url, client_builder.subcommand(vec!["python".to_owned(), "upgrade".to_owned()]), + &cache, args.default, globals.python_downloads, cli.top_level.no_config, diff --git a/crates/uv/tests/it/python_install.rs b/crates/uv/tests/it/python_install.rs index 98ca7756b..2d58a0788 100644 --- a/crates/uv/tests/it/python_install.rs +++ b/crates/uv/tests/it/python_install.rs @@ -2712,10 +2712,10 @@ fn python_install_no_cache() { - cpython-3.14.2-[PLATFORM] (python3.14) "); - // 3.12 isn't cached, so it can't be installed + // 3.12 isn't cached, so it can't be installed offline let mut filters = context.filters(); filters.push(( - "cpython-3.12.*.tar.gz", + r"cpython-3\.12\.\d+(%2B|\+)\d+-[a-z0-9_-]+\.tar\.gz", "cpython-3.12.[PATCH]-[DATE]-[PLATFORM].tar.gz", )); filters.push((r"releases/download/\d{8}/", "releases/download/[DATE]/")); @@ -2729,8 +2729,7 @@ fn python_install_no_cache() { ----- stderr ----- error: Failed to install cpython-3.12.12-[PLATFORM] - Caused by: Failed to download https://github.com/astral-sh/python-build-standalone/releases/download/[DATE]/cpython-3.12.[PATCH]-[DATE]-[PLATFORM].tar.gz - Caused by: Network connectivity is disabled, but the requested data wasn't found in the cache for: `https://github.com/astral-sh/python-build-standalone/releases/download/[DATE]/cpython-3.12.[PATCH]-[DATE]-[PLATFORM].tar.gz` + Caused by: An offline Python installation was requested, but cpython-3.12.12-[PLATFORM] (from https://github.com/astral-sh/python-build-standalone/releases/download/[DATE]/cpython-3.12.[PATCH]-[DATE]-[PLATFORM].tar.gz) is missing in [CACHE_DIR]/python-v0 "); }