From b9826778b989143ece02b2dee7a6cc6885515a7e Mon Sep 17 00:00:00 2001 From: Meitar Reihan Date: Sat, 15 Nov 2025 00:51:24 +0200 Subject: [PATCH] Support http/https URLs in `uv python --python-downloads-json-url` (#16542) continuation PR based on #14687 --------- Co-authored-by: Geoffrey Thomas Co-authored-by: Aria Desires --- Cargo.lock | 1 - Cargo.toml | 1 - crates/uv-cli/src/lib.rs | 8 - crates/uv-python/Cargo.toml | 1 - crates/uv-python/src/downloads.rs | 212 +++++++++++++++-------- crates/uv-python/src/installation.rs | 58 ++++--- crates/uv-settings/src/settings.rs | 2 - crates/uv-static/src/env_vars.rs | 7 +- crates/uv/src/commands/pip/compile.rs | 12 +- crates/uv/src/commands/python/find.rs | 7 +- crates/uv/src/commands/python/install.rs | 65 ++++--- crates/uv/src/commands/python/list.rs | 10 +- crates/uv/src/commands/python/pin.rs | 13 +- crates/uv/src/lib.rs | 2 + crates/uv/tests/it/common/mod.rs | 12 +- crates/uv/tests/it/help.rs | 4 +- crates/uv/tests/it/python_list.rs | 112 ++++++++++++ docs/reference/cli.md | 12 +- docs/reference/environment.md | 7 +- docs/reference/settings.md | 2 - uv.schema.json | 2 +- 21 files changed, 371 insertions(+), 179 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 628dc66eb..5d7f29678 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6398,7 +6398,6 @@ dependencies = [ "indoc", "insta", "itertools 0.14.0", - "once_cell", "owo-colors", "ref-cast", "regex", diff --git a/Cargo.toml b/Cargo.toml index 428088ece..8ff770e3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,7 +134,6 @@ memchr = { version = "2.7.4" } miette = { version = "7.2.0", features = ["fancy-no-backtrace"] } nanoid = { version = "0.4.0" } nix = { version = "0.30.0", features = ["signal"] } -once_cell = { version = "1.20.2" } open = { version = "5.3.2" } owo-colors = { version = "4.1.0" } path-slash = { version = "0.2.1" } diff --git a/crates/uv-cli/src/lib.rs b/crates/uv-cli/src/lib.rs index b4cb20290..488ac192b 100644 --- a/crates/uv-cli/src/lib.rs +++ b/crates/uv-cli/src/lib.rs @@ -5750,8 +5750,6 @@ pub struct PythonListArgs { pub output_format: PythonListFormat, /// URL pointing to JSON of custom Python installations. - /// - /// Note that currently, only local paths are supported. #[arg(long)] pub python_downloads_json_url: Option, } @@ -5848,8 +5846,6 @@ pub struct PythonInstallArgs { pub pypy_mirror: Option, /// URL pointing to JSON of custom Python installations. - /// - /// Note that currently, only local paths are supported. #[arg(long)] pub python_downloads_json_url: Option, @@ -5952,8 +5948,6 @@ pub struct PythonUpgradeArgs { pub reinstall: bool, /// URL pointing to JSON of custom Python installations. - /// - /// Note that currently, only local paths are supported. #[arg(long)] pub python_downloads_json_url: Option, } @@ -6034,8 +6028,6 @@ pub struct PythonFindArgs { pub show_version: bool, /// URL pointing to JSON of custom Python installations. - /// - /// Note that currently, only local paths are supported. #[arg(long)] pub python_downloads_json_url: Option, } diff --git a/crates/uv-python/Cargo.toml b/crates/uv-python/Cargo.toml index 7e48421b8..379328edf 100644 --- a/crates/uv-python/Cargo.toml +++ b/crates/uv-python/Cargo.toml @@ -66,7 +66,6 @@ tokio-util = { workspace = true, features = ["compat"] } tracing = { workspace = true } url = { workspace = true } which = { workspace = true } -once_cell = { workspace = true } [target.'cfg(target_os = "windows")'.dependencies] windows-registry = { workspace = true } diff --git a/crates/uv-python/src/downloads.rs b/crates/uv-python/src/downloads.rs index 1a892982b..f500d3e1d 100644 --- a/crates/uv-python/src/downloads.rs +++ b/crates/uv-python/src/downloads.rs @@ -10,13 +10,12 @@ use std::{env, io}; use futures::TryStreamExt; use itertools::Itertools; -use once_cell::sync::OnceCell; use owo_colors::OwoColorize; use reqwest_retry::policies::ExponentialBackoff; use reqwest_retry::{RetryError, RetryPolicy}; use serde::Deserialize; use thiserror::Error; -use tokio::io::{AsyncRead, AsyncWriteExt, BufWriter, ReadBuf}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt, BufWriter, ReadBuf}; use tokio_util::compat::FuturesAsyncReadCompatExt; use tokio_util::either::Either; use tracing::{debug, instrument}; @@ -102,10 +101,12 @@ pub enum Error { Mirror(&'static str, String), #[error("Failed to determine the libc used on the current platform")] LibcDetection(#[from] platform::LibcDetectionError), - #[error("Remote Python downloads JSON is not yet supported, please use a local path")] - RemoteJSONNotSupported, - #[error("The JSON of the python downloads is invalid: {0}")] - InvalidPythonDownloadsJSON(PathBuf, #[source] serde_json::Error), + #[error("Unable to parse the JSON Python download list at {0}")] + InvalidPythonDownloadsJSON(String, #[source] serde_json::Error), + #[error("This version of uv is too old to support the JSON Python download list at {0}")] + UnsupportedPythonDownloadsJSON(String), + #[error("Error while fetching remote python downloads json from '{0}'")] + FetchingPythonDownloadsJSONError(String, #[source] Box), #[error("An offline Python installation was requested, but {file} (from {url}) is missing in {}", python_builds_dir.user_display())] OfflinePythonMissing { file: Box, @@ -495,15 +496,6 @@ impl PythonDownloadRequest { } } - /// Iterate over all [`PythonDownload`]'s that match this request. - pub fn iter_downloads<'a>( - &'a self, - python_downloads_json_url: Option<&'a str>, - ) -> Result + use<'a>, Error> { - Ok(ManagedPythonDownload::iter_all(python_downloads_json_url)? - .filter(move |download| self.satisfied_by_download(download))) - } - /// Whether this request is satisfied by an installation key. pub fn satisfied_by_key(&self, key: &PythonInstallationKey) -> bool { // Check platform requirements @@ -900,10 +892,12 @@ impl FromStr for PythonDownloadRequest { } } -const BUILTIN_PYTHON_DOWNLOADS_JSON: &str = - include_str!(concat!(env!("OUT_DIR"), "/download-metadata-minified.json")); -static PYTHON_DOWNLOADS: OnceCell> = - OnceCell::new(); +const BUILTIN_PYTHON_DOWNLOADS_JSON: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/download-metadata-minified.json")); + +pub struct ManagedPythonDownloadList { + downloads: Vec, +} #[derive(Debug, Deserialize, Clone)] struct JsonPythonDownload { @@ -946,29 +940,33 @@ impl Display for ManagedPythonDownloadWithBuild<'_> { } } -impl ManagedPythonDownload { - /// Return a display type that includes the build information. - pub fn to_display_with_build(&self) -> ManagedPythonDownloadWithBuild<'_> { - ManagedPythonDownloadWithBuild(self) +impl ManagedPythonDownloadList { + /// Iterate over all [`ManagedPythonDownload`]s. + fn iter_all(&self) -> impl Iterator { + self.downloads.iter() + } + + /// Iterate over all [`ManagedPythonDownload`]s that match the request. + pub fn iter_matching( + &self, + request: &PythonDownloadRequest, + ) -> impl Iterator { + self.iter_all() + .filter(move |download| request.satisfied_by_download(download)) } /// Return the first [`ManagedPythonDownload`] matching a request, if any. /// /// If there is no stable version matching the request, a compatible pre-release version will /// be searched for — even if a pre-release was not explicitly requested. - pub fn from_request( - request: &PythonDownloadRequest, - python_downloads_json_url: Option<&str>, - ) -> Result<&'static Self, Error> { - if let Some(download) = request.iter_downloads(python_downloads_json_url)?.next() { + pub fn find(&self, request: &PythonDownloadRequest) -> Result<&ManagedPythonDownload, Error> { + if let Some(download) = self.iter_matching(request).next() { return Ok(download); } if !request.allows_prereleases() { - if let Some(download) = request - .clone() - .with_prereleases(true) - .iter_downloads(python_downloads_json_url)? + if let Some(download) = self + .iter_matching(&request.clone().with_prereleases(true)) .next() { return Ok(download); @@ -977,47 +975,107 @@ impl ManagedPythonDownload { Err(Error::NoDownloadFound(request.clone())) } - //noinspection RsUnresolvedPath - RustRover can't see through the `include!` - /// Iterate over all [`ManagedPythonDownload`]s. + /// Load available Python distributions from a provided source or the compiled-in list. /// - /// Note: The list is generated on the first call to this function. - /// so `python_downloads_json_url` is only used in the first call to this function. - pub fn iter_all( + /// `python_downloads_json_url` can be either `None`, to use the default list (taken from + /// `crates/uv-python/download-metadata.json`), or `Some` local path + /// or file://, http://, or https:// URL. + /// + /// Returns an error if the provided list could not be opened, if the JSON is invalid, or if it + /// does not parse into the expected data structure. + pub async fn new( + client: &BaseClient, python_downloads_json_url: Option<&str>, - ) -> Result, Error> { - let downloads = PYTHON_DOWNLOADS.get_or_try_init(|| { - let json_downloads: HashMap = if let Some(json_source) = - python_downloads_json_url - { - // Windows paths are also valid URLs - let json_source = if let Ok(url) = Url::parse(json_source) { - if let Ok(path) = url.to_file_path() { - Cow::Owned(path) - } else if matches!(url.scheme(), "http" | "https") { - return Err(Error::RemoteJSONNotSupported); - } else { - Cow::Borrowed(Path::new(json_source)) - } - } else { - Cow::Borrowed(Path::new(json_source)) - }; + ) -> Result { + // Although read_url() handles file:// URLs and converts them to local file reads, here we + // want to also support parsing bare filenames like "/tmp/py.json", not just + // "file:///tmp/py.json". Note that "C:\Temp\py.json" should be considered a filename, even + // though Url::parse would successfully misparse it as a URL with scheme "C". + enum Source<'a> { + BuiltIn, + Path(Cow<'a, Path>), + Http(DisplaySafeUrl), + } - let file = fs_err::File::open(json_source.as_ref())?; - - serde_json::from_reader(file) - .map_err(|e| Error::InvalidPythonDownloadsJSON(json_source.to_path_buf(), e))? + let json_source = if let Some(url_or_path) = python_downloads_json_url { + if let Ok(url) = DisplaySafeUrl::parse(url_or_path) { + match url.scheme() { + "http" | "https" => Source::Http(url), + "file" => Source::Path(Cow::Owned( + url.to_file_path().or(Err(Error::InvalidUrlFormat(url)))?, + )), + _ => Source::Path(Cow::Borrowed(Path::new(url_or_path))), + } } else { - serde_json::from_str(BUILTIN_PYTHON_DOWNLOADS_JSON).map_err(|e| { - Error::InvalidPythonDownloadsJSON(PathBuf::from("EMBEDDED IN THE BINARY"), e) - })? - }; + Source::Path(Cow::Borrowed(Path::new(url_or_path))) + } + } else { + Source::BuiltIn + }; - let result = parse_json_downloads(json_downloads); - Ok(Cow::Owned(result)) - })?; + let buf: Cow<'_, [u8]> = match json_source { + Source::BuiltIn => BUILTIN_PYTHON_DOWNLOADS_JSON.into(), + Source::Path(ref path) => fs_err::read(path.as_ref())?.into(), + Source::Http(ref url) => fetch_bytes_from_url(client, url) + .await + .map_err(|e| Error::FetchingPythonDownloadsJSONError(url.to_string(), Box::new(e)))? + .into(), + }; + let json_downloads: HashMap = serde_json::from_slice(&buf) + .map_err( + // As an explicit compatibility mechanism, if there's a top-level "version" key, it + // means it's a newer format than we know how to deal with. Before reporting a + // parse error about the format of JsonPythonDownload, check for that key. We can do + // this by parsing into a Map which allows any valid JSON on the + // value side. (Because it's zero-sized, Clippy suggests Set, but that won't + // have the same parsing effect.) + #[allow(clippy::zero_sized_map_values)] + |e| { + let source = match json_source { + Source::BuiltIn => "EMBEDDED IN THE BINARY".to_owned(), + Source::Path(path) => path.to_string_lossy().to_string(), + Source::Http(url) => url.to_string(), + }; + if let Ok(keys) = + serde_json::from_slice::>(&buf) + && keys.contains_key("version") + { + Error::UnsupportedPythonDownloadsJSON(source) + } else { + Error::InvalidPythonDownloadsJSON(source, e) + } + }, + )?; - Ok(downloads.iter()) + let result = parse_json_downloads(json_downloads); + Ok(Self { downloads: result }) + } + + /// Load available Python distributions from the compiled-in list only. + /// for testing purposes. + pub fn new_only_embedded() -> Result { + let json_downloads: HashMap = + serde_json::from_slice(BUILTIN_PYTHON_DOWNLOADS_JSON).map_err(|e| { + Error::InvalidPythonDownloadsJSON("EMBEDDED IN THE BINARY".to_owned(), e) + })?; + let result = parse_json_downloads(json_downloads); + Ok(Self { downloads: result }) + } +} + +async fn fetch_bytes_from_url(client: &BaseClient, url: &DisplaySafeUrl) -> Result, Error> { + let (mut reader, size) = read_url(url, client).await?; + let capacity = size.and_then(|s| s.try_into().ok()).unwrap_or(1_048_576); + let mut buf = Vec::with_capacity(capacity); + reader.read_to_end(&mut buf).await?; + Ok(buf) +} + +impl ManagedPythonDownload { + /// Return a display type that includes the build information. + pub fn to_display_with_build(&self) -> ManagedPythonDownloadWithBuild<'_> { + ManagedPythonDownloadWithBuild(self) } pub fn url(&self) -> &Cow<'static, str> { @@ -1925,15 +1983,18 @@ mod tests { } /// Test that build filtering works correctly - #[test] - fn test_python_download_request_build_filtering() { + #[tokio::test] + async fn test_python_download_request_build_filtering() { let request = PythonDownloadRequest::default() .with_version(VersionRequest::from_str("3.12").unwrap()) .with_implementation(ImplementationName::CPython) .with_build("20240814".to_string()); - let downloads: Vec<_> = ManagedPythonDownload::iter_all(None) - .unwrap() + let client = uv_client::BaseClientBuilder::default().build(); + let download_list = ManagedPythonDownloadList::new(&client, None).await.unwrap(); + + let downloads: Vec<_> = download_list + .iter_all() .filter(|d| request.satisfied_by_download(d)) .collect(); @@ -1947,17 +2008,20 @@ mod tests { } /// Test that an invalid build results in no matches - #[test] - fn test_python_download_request_invalid_build() { + #[tokio::test] + async fn test_python_download_request_invalid_build() { // Create a request with a non-existent build let request = PythonDownloadRequest::default() .with_version(VersionRequest::from_str("3.12").unwrap()) .with_implementation(ImplementationName::CPython) .with_build("99999999".to_string()); + let client = uv_client::BaseClientBuilder::default().build(); + let download_list = ManagedPythonDownloadList::new(&client, None).await.unwrap(); + // Should find no matching downloads - let downloads: Vec<_> = ManagedPythonDownload::iter_all(None) - .unwrap() + let downloads: Vec<_> = download_list + .iter_all() .filter(|d| request.satisfied_by_download(d)) .collect(); diff --git a/crates/uv-python/src/installation.rs b/crates/uv-python/src/installation.rs index 065336441..2cc751bda 100644 --- a/crates/uv-python/src/installation.rs +++ b/crates/uv-python/src/installation.rs @@ -5,11 +5,12 @@ use std::str::FromStr; use indexmap::IndexMap; use ref_cast::RefCast; +use reqwest_retry::policies::ExponentialBackoff; use tracing::{debug, info}; use uv_warnings::warn_user; use uv_cache::Cache; -use uv_client::BaseClientBuilder; +use uv_client::{BaseClient, BaseClientBuilder}; use uv_pep440::{Prerelease, Version}; use uv_platform::{Arch, Libc, Os, Platform}; use uv_preview::Preview; @@ -17,7 +18,10 @@ use uv_preview::Preview; use crate::discovery::{ EnvironmentPreference, PythonRequest, find_best_python_installation, find_python_installation, }; -use crate::downloads::{DownloadResult, ManagedPythonDownload, PythonDownloadRequest, Reporter}; +use crate::downloads::{ + DownloadResult, ManagedPythonDownload, ManagedPythonDownloadList, PythonDownloadRequest, + Reporter, +}; use crate::implementation::LenientImplementationName; use crate::managed::{ManagedPythonInstallation, ManagedPythonInstallations}; use crate::{ @@ -59,13 +63,13 @@ impl PythonInstallation { request: &PythonRequest, environments: EnvironmentPreference, preference: PythonPreference, - python_downloads_json_url: Option<&str>, + download_list: &ManagedPythonDownloadList, cache: &Cache, preview: Preview, ) -> Result { let installation = find_python_installation(request, environments, preference, cache, preview)??; - installation.warn_if_outdated_prerelease(request, python_downloads_json_url); + installation.warn_if_outdated_prerelease(request, download_list); Ok(installation) } @@ -75,13 +79,13 @@ impl PythonInstallation { request: &PythonRequest, environments: EnvironmentPreference, preference: PythonPreference, - python_downloads_json_url: Option<&str>, + download_list: &ManagedPythonDownloadList, cache: &Cache, preview: Preview, ) -> Result { let installation = find_best_python_installation(request, environments, preference, cache, preview)??; - installation.warn_if_outdated_prerelease(request, python_downloads_json_url); + installation.warn_if_outdated_prerelease(request, download_list); Ok(installation) } @@ -103,12 +107,19 @@ impl PythonInstallation { ) -> Result { let request = request.unwrap_or(&PythonRequest::Default); + // Python downloads are performing their own retries to catch stream errors, disable the + // default retries to avoid the middleware performing uncontrolled retries. + let retry_policy = client_builder.retry_policy(); + let client = client_builder.clone().retries(0).build(); + let download_list = + ManagedPythonDownloadList::new(&client, python_downloads_json_url).await?; + // Search for the installation let err = match Self::find( request, environments, preference, - python_downloads_json_url, + &download_list, cache, preview, ) { @@ -134,9 +145,10 @@ impl PythonInstallation { && python_downloads.is_automatic() && client_builder.connectivity.is_online(); - let download = download_request.clone().fill().map(|request| { - ManagedPythonDownload::from_request(&request, python_downloads_json_url) - }); + let download = download_request + .clone() + .fill() + .map(|request| download_list.find(&request)); // Regardless of whether downloads are enabled, we want to determine if the download is // available to power error messages. However, if downloads aren't enabled, we don't want to @@ -211,7 +223,8 @@ impl PythonInstallation { let installation = Self::fetch( download, - client_builder, + &client, + &retry_policy, cache, reporter, python_install_mirror, @@ -220,15 +233,16 @@ impl PythonInstallation { ) .await?; - installation.warn_if_outdated_prerelease(request, python_downloads_json_url); + installation.warn_if_outdated_prerelease(request, &download_list); Ok(installation) } /// Download and install the requested installation. pub async fn fetch( - download: &'static ManagedPythonDownload, - client_builder: &BaseClientBuilder<'_>, + download: &ManagedPythonDownload, + client: &BaseClient, + retry_policy: &ExponentialBackoff, cache: &Cache, reporter: Option<&dyn Reporter>, python_install_mirror: Option<&str>, @@ -240,16 +254,11 @@ impl PythonInstallation { let scratch_dir = installations.scratch(); let _lock = installations.lock().await?; - // Python downloads are performing their own retries to catch stream errors, disable the - // default retries to avoid the middleware from performing uncontrolled retries. - let retry_policy = client_builder.retry_policy(); - let client = client_builder.clone().retries(0).build(); - info!("Fetching requested Python..."); let result = download .fetch_with_retry( - &client, - &retry_policy, + client, + retry_policy, installations_dir, &scratch_dir, false, @@ -361,7 +370,7 @@ impl PythonInstallation { pub(crate) fn warn_if_outdated_prerelease( &self, request: &PythonRequest, - python_downloads_json_url: Option<&str>, + download_list: &ManagedPythonDownloadList, ) { if request.allows_prereleases() { return; @@ -398,10 +407,7 @@ impl PythonInstallation { let download_request = download_request.with_prereleases(false); let has_stable_download = { - let Ok(mut downloads) = download_request.iter_downloads(python_downloads_json_url) - else { - return; - }; + let mut downloads = download_list.iter_matching(&download_request); downloads.any(|download| { let download_version = download.key().version().into_version(); diff --git a/crates/uv-settings/src/settings.rs b/crates/uv-settings/src/settings.rs index 32901020b..c25d5f0ed 100644 --- a/crates/uv-settings/src/settings.rs +++ b/crates/uv-settings/src/settings.rs @@ -997,8 +997,6 @@ pub struct PythonInstallMirrors { pub pypy_install_mirror: Option, /// URL pointing to JSON of custom Python installations. - /// - /// Note that currently, only local paths are supported. #[option( default = "None", value_type = "str", diff --git a/crates/uv-static/src/env_vars.rs b/crates/uv-static/src/env_vars.rs index 60c27579c..53e1d98f1 100644 --- a/crates/uv-static/src/env_vars.rs +++ b/crates/uv-static/src/env_vars.rs @@ -400,10 +400,11 @@ impl EnvVars { /// Managed Python installations information is hardcoded in the `uv` binary. /// - /// This variable can be set to a URL pointing to JSON to use as a list for Python installations. - /// This will allow for setting each property of the Python installation, mostly the url part for offline mirror. + /// This variable can be set to a local path or URL pointing to + /// a JSON list of Python installations to override the hardcoded list. /// - /// Note that currently, only local paths are supported. + /// This allows customizing the URLs for downloads or using slightly older or newer versions + /// of Python than the ones hardcoded into this build of `uv`. #[attr_added_in("0.6.13")] pub const UV_PYTHON_DOWNLOADS_JSON_URL: &'static str = "UV_PYTHON_DOWNLOADS_JSON_URL"; diff --git a/crates/uv/src/commands/pip/compile.rs b/crates/uv/src/commands/pip/compile.rs index e5779bcdd..b1c5501d5 100644 --- a/crates/uv/src/commands/pip/compile.rs +++ b/crates/uv/src/commands/pip/compile.rs @@ -10,6 +10,8 @@ use owo_colors::OwoColorize; use rustc_hash::FxHashSet; use tracing::debug; +use uv_python::downloads::ManagedPythonDownloadList; + use uv_cache::Cache; use uv_client::{BaseClientBuilder, FlatIndexClient, RegistryClientBuilder}; use uv_configuration::{ @@ -292,13 +294,19 @@ pub(crate) async fn pip_compile( // Find an interpreter to use for building distributions let environment_preference = EnvironmentPreference::from_system_flag(system, false); let python_preference = python_preference.with_system_flag(system); + let client = client_builder.clone().retries(0).build(); + let download_list = ManagedPythonDownloadList::new( + &client, + install_mirrors.python_downloads_json_url.as_deref(), + ) + .await?; let interpreter = if let Some(python) = python.as_ref() { let request = PythonRequest::parse(python); PythonInstallation::find( &request, environment_preference, python_preference, - install_mirrors.python_downloads_json_url.as_deref(), + &download_list, &cache, preview, ) @@ -315,7 +323,7 @@ pub(crate) async fn pip_compile( &request, environment_preference, python_preference, - install_mirrors.python_downloads_json_url.as_deref(), + &download_list, &cache, preview, ) diff --git a/crates/uv/src/commands/python/find.rs b/crates/uv/src/commands/python/find.rs index 4df03d85d..31e28e6a7 100644 --- a/crates/uv/src/commands/python/find.rs +++ b/crates/uv/src/commands/python/find.rs @@ -7,6 +7,7 @@ use uv_client::BaseClientBuilder; use uv_configuration::DependencyGroupsWithDefaults; use uv_fs::Simplified; use uv_preview::Preview; +use uv_python::downloads::ManagedPythonDownloadList; use uv_python::{ EnvironmentPreference, PythonDownloads, PythonInstallation, PythonPreference, PythonRequest, }; @@ -32,6 +33,7 @@ pub(crate) async fn find( system: bool, python_preference: PythonPreference, python_downloads_json_url: Option<&str>, + client_builder: &BaseClientBuilder<'_>, cache: &Cache, printer: Printer, preview: Preview, @@ -75,11 +77,14 @@ pub(crate) async fn find( ) .await?; + let client = client_builder.clone().retries(0).build(); + let download_list = ManagedPythonDownloadList::new(&client, python_downloads_json_url).await?; + let python = PythonInstallation::find( &python_request.unwrap_or_default(), environment_preference, python_preference, - python_downloads_json_url, + &download_list, cache, preview, )?; diff --git a/crates/uv/src/commands/python/install.rs b/crates/uv/src/commands/python/install.rs index 10e9480f7..480731cbe 100644 --- a/crates/uv/src/commands/python/install.rs +++ b/crates/uv/src/commands/python/install.rs @@ -19,7 +19,8 @@ use uv_fs::Simplified; use uv_platform::{Arch, Libc}; use uv_preview::{Preview, PreviewFeatures}; use uv_python::downloads::{ - self, ArchRequest, DownloadResult, ManagedPythonDownload, PythonDownloadRequest, + self, ArchRequest, DownloadResult, ManagedPythonDownload, ManagedPythonDownloadList, + PythonDownloadRequest, }; use uv_python::managed::{ ManagedPythonInstallation, ManagedPythonInstallations, PythonMinorVersionLink, @@ -39,17 +40,17 @@ use crate::commands::{ExitStatus, elapsed}; use crate::printer::Printer; #[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct InstallRequest { +struct InstallRequest<'a> { /// The original request from the user request: PythonRequest, /// A download request corresponding to the `request` with platform information filled download_request: PythonDownloadRequest, /// A download that satisfies the request - download: &'static ManagedPythonDownload, + download: &'a ManagedPythonDownload, } -impl InstallRequest { - fn new(request: PythonRequest, python_downloads_json_url: Option<&str>) -> Result { +impl<'a> InstallRequest<'a> { + fn new(request: PythonRequest, download_list: &'a ManagedPythonDownloadList) -> Result { // Make sure the request is a valid download request and fill platform information let download_request = PythonDownloadRequest::from_request(&request) .ok_or_else(|| { @@ -61,22 +62,20 @@ impl InstallRequest { .fill()?; // Find a matching download - let download = - match ManagedPythonDownload::from_request(&download_request, python_downloads_json_url) + let download = match download_list.find(&download_request) { + Ok(download) => download, + Err(downloads::Error::NoDownloadFound(request)) + if request.libc().is_some_and(Libc::is_musl) + && request + .arch() + .is_some_and(|arch| Arch::is_arm(&arch.inner())) => { - Ok(download) => download, - Err(downloads::Error::NoDownloadFound(request)) - if request.libc().is_some_and(Libc::is_musl) - && request - .arch() - .is_some_and(|arch| Arch::is_arm(&arch.inner())) => - { - return Err(anyhow::anyhow!( - "uv does not yet provide musl Python distributions on aarch64." - )); - } - Err(err) => return Err(err.into()), - }; + return Err(anyhow::anyhow!( + "uv does not yet provide musl Python distributions on aarch64." + )); + } + Err(err) => return Err(err.into()), + }; Ok(Self { request, @@ -94,7 +93,7 @@ impl InstallRequest { } } -impl std::fmt::Display for InstallRequest { +impl std::fmt::Display for InstallRequest<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let request = self.request.to_canonical_string(); let download = self.download_request.to_string(); @@ -234,6 +233,12 @@ pub(crate) async fn install( // Resolve the requests let mut is_default_install = false; let mut is_unspecified_upgrade = false; + let retry_policy = client_builder.retry_policy(); + // Python downloads are performing their own retries to catch stream errors, disable the + // default retries to avoid the middleware from performing uncontrolled retries. + let client = client_builder.retries(0).build(); + let download_list = + ManagedPythonDownloadList::new(&client, python_downloads_json_url.as_deref()).await?; // TODO(zanieb): We use this variable to special-case .python-version files, but it'd be nice to // have generalized request source tracking instead let mut is_from_python_version_file = false; @@ -251,10 +256,8 @@ pub(crate) async fn install( let version = request.take_version().unwrap(); // Drop the patch and prerelease parts from the request request = request.with_version(version.only_minor()); - let install_request = InstallRequest::new( - PythonRequest::Key(request), - python_downloads_json_url.as_deref(), - )?; + let install_request = + InstallRequest::new(PythonRequest::Key(request), &download_list)?; minor_version_requests.insert(install_request); } minor_version_requests.into_iter().collect::>() @@ -287,14 +290,14 @@ pub(crate) async fn install( }] }) .into_iter() - .map(|request| InstallRequest::new(request, python_downloads_json_url.as_deref())) + .map(|request| InstallRequest::new(request, &download_list)) .collect::>>()? } } else { targets .iter() .map(|target| PythonRequest::parse(target.as_str())) - .map(|request| InstallRequest::new(request, python_downloads_json_url.as_deref())) + .map(|request| InstallRequest::new(request, &download_list)) .collect::>>()? }; @@ -377,7 +380,7 @@ pub(crate) async fn install( // Construct an install request matching the existing installation match InstallRequest::new( PythonRequest::Key(installation.into()), - python_downloads_json_url.as_deref(), + &download_list, ) { Ok(request) => { debug!("Will reinstall `{}`", installation.key()); @@ -453,11 +456,7 @@ pub(crate) async fn install( .unique_by(|download| download.key()) .collect::>(); - let retry_policy = client_builder.retry_policy(); - // Python downloads are performing their own retries to catch stream errors, disable the - // default retries to avoid the middleware from performing uncontrolled retries. - let client = client_builder.retries(0).build(); - + // Download and unpack the Python versions concurrently let reporter = PythonDownloadReporter::new(printer, Some(downloads.len() as u64)); let mut tasks = FuturesUnordered::new(); diff --git a/crates/uv/src/commands/python/list.rs b/crates/uv/src/commands/python/list.rs index 722268539..96c500e35 100644 --- a/crates/uv/src/commands/python/list.rs +++ b/crates/uv/src/commands/python/list.rs @@ -10,8 +10,9 @@ use itertools::Either; use owo_colors::OwoColorize; use rustc_hash::FxHashSet; use uv_cache::Cache; +use uv_client::BaseClientBuilder; use uv_fs::Simplified; -use uv_python::downloads::PythonDownloadRequest; +use uv_python::downloads::{ManagedPythonDownloadList, PythonDownloadRequest}; use uv_python::{ DiscoveryError, EnvironmentPreference, PythonDownloads, PythonInstallation, PythonNotFound, PythonPreference, PythonRequest, PythonSource, find_python_installations, @@ -63,6 +64,7 @@ pub(crate) async fn list( python_downloads_json_url: Option, python_preference: PythonPreference, python_downloads: PythonDownloads, + client_builder: &BaseClientBuilder<'_>, cache: &Cache, printer: Printer, preview: Preview, @@ -75,6 +77,9 @@ pub(crate) async fn list( PythonDownloadRequest::from_request(request.as_ref().unwrap_or(&PythonRequest::Any)) }; + let client = client_builder.build(); + let download_list = + ManagedPythonDownloadList::new(&client, python_downloads_json_url.as_deref()).await?; let mut output = BTreeSet::new(); if let Some(base_download_request) = base_download_request { let download_request = match kinds { @@ -106,8 +111,7 @@ pub(crate) async fn list( let downloads = download_request .as_ref() - .map(|a| PythonDownloadRequest::iter_downloads(a, python_downloads_json_url.as_deref())) - .transpose()? + .map(|request| download_list.iter_matching(request)) .into_iter() .flatten() // TODO(zanieb): Add a way to show debug downloads, we just hide them for now diff --git a/crates/uv/src/commands/python/pin.rs b/crates/uv/src/commands/python/pin.rs index ef5585036..6f04ef854 100644 --- a/crates/uv/src/commands/python/pin.rs +++ b/crates/uv/src/commands/python/pin.rs @@ -5,6 +5,7 @@ use std::str::FromStr; use anyhow::{Result, bail}; use owo_colors::OwoColorize; use tracing::debug; +use uv_python::downloads::ManagedPythonDownloadList; use uv_cache::Cache; use uv_client::BaseClientBuilder; @@ -96,11 +97,17 @@ pub(crate) async fn pin( for pin in file.versions() { writeln!(printer.stdout(), "{}", pin.to_canonical_string())?; if let Some(virtual_project) = &virtual_project { + let client = client_builder.clone().retries(0).build(); + let download_list = ManagedPythonDownloadList::new( + &client, + install_mirrors.python_downloads_json_url.as_deref(), + ) + .await?; warn_if_existing_pin_incompatible_with_project( pin, virtual_project, python_preference, - install_mirrors.python_downloads_json_url.as_deref(), + &download_list, cache, preview, ); @@ -265,7 +272,7 @@ fn warn_if_existing_pin_incompatible_with_project( pin: &PythonRequest, virtual_project: &VirtualProject, python_preference: PythonPreference, - python_downloads_json_url: Option<&str>, + downloads_list: &ManagedPythonDownloadList, cache: &Cache, preview: Preview, ) { @@ -291,7 +298,7 @@ fn warn_if_existing_pin_incompatible_with_project( pin, EnvironmentPreference::OnlySystem, python_preference, - python_downloads_json_url, + downloads_list, cache, preview, ) { diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index 00a572a9b..b1a1a2711 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -1532,6 +1532,7 @@ async fn run(mut cli: Cli) -> Result { args.python_downloads_json_url, globals.python_preference, globals.python_downloads, + &client_builder, &cache, printer, globals.preview, @@ -1643,6 +1644,7 @@ async fn run(mut cli: Cli) -> Result { args.system, globals.python_preference, args.python_downloads_json_url.as_deref(), + &client_builder, &cache, printer, globals.preview, diff --git a/crates/uv/tests/it/common/mod.rs b/crates/uv/tests/it/common/mod.rs index 07cd37085..40a131e10 100644 --- a/crates/uv/tests/it/common/mod.rs +++ b/crates/uv/tests/it/common/mod.rs @@ -8,6 +8,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, ExitStatus, Output}; use std::str::FromStr; use std::{env, io}; +use uv_python::downloads::ManagedPythonDownloadList; use assert_cmd::assert::{Assert, OutputAssertExt}; use assert_fs::assert::PathAssert; @@ -627,11 +628,13 @@ impl TestContext { .expect("CARGO_MANIFEST_DIR should be doubly nested in workspace") .to_path_buf(); + let download_list = ManagedPythonDownloadList::new_only_embedded().unwrap(); + let python_versions: Vec<_> = python_versions .iter() .map(|version| PythonVersion::from_str(version).unwrap()) .zip( - python_installations_for_versions(&temp_dir, python_versions, None) + python_installations_for_versions(&temp_dir, python_versions, &download_list) .expect("Failed to find test Python versions"), ) .collect(); @@ -1688,8 +1691,9 @@ pub fn python_path_with_versions( temp_dir: &ChildPath, python_versions: &[&str], ) -> anyhow::Result { + let download_list = ManagedPythonDownloadList::new_only_embedded().unwrap(); Ok(env::join_paths( - python_installations_for_versions(temp_dir, python_versions, None)? + python_installations_for_versions(temp_dir, python_versions, &download_list)? .into_iter() .map(|path| path.parent().unwrap().to_path_buf()), )?) @@ -1701,7 +1705,7 @@ pub fn python_path_with_versions( pub fn python_installations_for_versions( temp_dir: &ChildPath, python_versions: &[&str], - python_downloads_json_url: Option<&str>, + download_list: &ManagedPythonDownloadList, ) -> anyhow::Result> { let cache = Cache::from_path(temp_dir.child("cache").to_path_buf()).init()?; let selected_pythons = python_versions @@ -1711,7 +1715,7 @@ pub fn python_installations_for_versions( &PythonRequest::parse(python_version), EnvironmentPreference::OnlySystem, PythonPreference::Managed, - python_downloads_json_url, + download_list, &cache, Preview::default(), ) { diff --git a/crates/uv/tests/it/help.rs b/crates/uv/tests/it/help.rs index 648a34ece..e6e5fd801 100644 --- a/crates/uv/tests/it/help.rs +++ b/crates/uv/tests/it/help.rs @@ -541,9 +541,7 @@ fn help_subsubcommand() { Distributions can be read from a local directory by using the `file://` URL scheme. --python-downloads-json-url - URL pointing to JSON of custom Python installations. - - Note that currently, only local paths are supported. + URL pointing to JSON of custom Python installations -r, --reinstall Reinstall the requested Python version, if it's already installed. diff --git a/crates/uv/tests/it/python_list.rs b/crates/uv/tests/it/python_list.rs index 706480ec4..7b77da6e1 100644 --- a/crates/uv/tests/it/python_list.rs +++ b/crates/uv/tests/it/python_list.rs @@ -2,6 +2,11 @@ use uv_platform::{Arch, Os}; use uv_static::EnvVars; use crate::common::{TestContext, uv_snapshot}; +use anyhow::Result; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, +}; #[test] fn python_list() { @@ -479,3 +484,110 @@ fn python_list_downloads_installed() { ----- stderr ----- "); } + +#[tokio::test] +async fn python_list_remote_python_downloads_json_url() -> Result<()> { + let context: TestContext = TestContext::new_with_versions(&[]); + let server = MockServer::start().await; + + let remote_json = r#" + { + "cpython-3.14.0-darwin-aarch64-none": { + "name": "cpython", + "arch": { + "family": "aarch64", + "variant": null + }, + "os": "darwin", + "libc": "none", + "major": 3, + "minor": 14, + "patch": 0, + "prerelease": "", + "url": "https://custom.com/cpython-3.14.0-darwin-aarch64-none.tar.gz", + "sha256": "c3223d5924a0ed0ef5958a750377c362d0957587f896c0f6c635ae4b39e0f337", + "variant": null, + "build": "20251028" + }, + "cpython-3.13.2+freethreaded-linux-powerpc64le-gnu": { + "name": "cpython", + "arch": { + "family": "powerpc64le", + "variant": null + }, + "os": "linux", + "libc": "gnu", + "major": 3, + "minor": 13, + "patch": 2, + "prerelease": "", + "url": "https://custom.com/ccpython-3.13.2+freethreaded-linux-powerpc64le-gnu.tar.gz", + "sha256": "6ae8fa44cb2edf4ab49cff1820b53c40c10349c0f39e11b8cd76ce7f3e7e1def", + "variant": "freethreaded", + "build": "20250317" + } + } + "#; + Mock::given(method("GET")) + .and(path("/")) + .respond_with(ResponseTemplate::new(200).set_body_raw(remote_json, "application/json")) + .mount(&server) + .await; + + Mock::given(method("GET")) + .and(path("/invalid")) + .respond_with(ResponseTemplate::new(200).set_body_raw("{", "application/json")) + .mount(&server) + .await; + + // Test showing all interpreters from the remote JSON URL + uv_snapshot!(context + .python_list() + .env_remove(EnvVars::UV_PYTHON_DOWNLOADS) + .arg("--all-versions") + .arg("--all-platforms") + .arg("--all-arches") + .arg("--show-urls") + .arg("--python-downloads-json-url").arg(server.uri()), @r" + success: true + exit_code: 0 + ----- stdout ----- + cpython-3.14.0-macos-aarch64-none https://custom.com/cpython-3.14.0-darwin-aarch64-none.tar.gz + cpython-3.13.2+freethreaded-linux-powerpc64le-gnu https://custom.com/ccpython-3.13.2+freethreaded-linux-powerpc64le-gnu.tar.gz + + ----- stderr ----- + "); + + // test invalid URL path + uv_snapshot!(context.filters(), context + .python_list() + .env_remove(EnvVars::UV_PYTHON_DOWNLOADS) + .arg("--python-downloads-json-url").arg(format!("{}/404", server.uri())), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: Error while fetching remote python downloads json from 'http://[LOCALHOST]/404' + Caused by: Failed to download http://[LOCALHOST]/404 + Caused by: HTTP status client error (404 Not Found) for url (http://[LOCALHOST]/404) + + "); + + // test invalid json + uv_snapshot!(context.filters(), context + .python_list() + .env_remove(EnvVars::UV_PYTHON_DOWNLOADS) + .arg("--python-downloads-json-url").arg(format!("{}/invalid", server.uri())), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: Unable to parse the JSON Python download list at http://[LOCALHOST]/invalid + Caused by: EOF while parsing an object at line 1 column 1 + + "); + + Ok(()) +} diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 25b0e10ba..450978ca5 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -3420,8 +3420,7 @@ uv python list [OPTIONS] [REQUEST]

Other command-line arguments (such as relative paths) will be resolved relative to the current working directory.

See --directory to change the working directory entirely.

This setting has no effect when used in the uv pip interface.

-

May also be set with the UV_PROJECT environment variable.

--python-downloads-json-url python-downloads-json-url

URL pointing to JSON of custom Python installations.

-

Note that currently, only local paths are supported.

+

May also be set with the UV_PROJECT environment variable.

--python-downloads-json-url python-downloads-json-url

URL pointing to JSON of custom Python installations

--quiet, -q

Use quiet output.

Repeating this option, e.g., -qq, will enable a silent mode in which uv will write no output to stdout.

--show-urls

Show the URLs of available Python downloads.

@@ -3519,8 +3518,7 @@ uv python install [OPTIONS] [TARGETS]...

May also be set with the UV_PROJECT environment variable.

--pypy-mirror pypy-mirror

Set the URL to use as the source for downloading PyPy installations.

The provided URL will replace https://downloads.python.org/pypy in, e.g., https://downloads.python.org/pypy/pypy3.8-v7.3.7-osx64.tar.bz2.

Distributions can be read from a local directory by using the file:// URL scheme.

-
--python-downloads-json-url python-downloads-json-url

URL pointing to JSON of custom Python installations.

-

Note that currently, only local paths are supported.

+
--python-downloads-json-url python-downloads-json-url

URL pointing to JSON of custom Python installations

--quiet, -q

Use quiet output.

Repeating this option, e.g., -qq, will enable a silent mode in which uv will write no output to stdout.

--reinstall, -r

Reinstall the requested Python version, if it's already installed.

@@ -3612,8 +3610,7 @@ uv python upgrade [OPTIONS] [TARGETS]...

May also be set with the UV_PROJECT environment variable.

--pypy-mirror pypy-mirror

Set the URL to use as the source for downloading PyPy installations.

The provided URL will replace https://downloads.python.org/pypy in, e.g., https://downloads.python.org/pypy/pypy3.8-v7.3.7-osx64.tar.bz2.

Distributions can be read from a local directory by using the file:// URL scheme.

-
--python-downloads-json-url python-downloads-json-url

URL pointing to JSON of custom Python installations.

-

Note that currently, only local paths are supported.

+
--python-downloads-json-url python-downloads-json-url

URL pointing to JSON of custom Python installations

--quiet, -q

Use quiet output.

Repeating this option, e.g., -qq, will enable a silent mode in which uv will write no output to stdout.

--reinstall, -r

Reinstall the latest Python patch, if it's already installed.

@@ -3686,8 +3683,7 @@ uv python find [OPTIONS] [REQUEST]

Other command-line arguments (such as relative paths) will be resolved relative to the current working directory.

See --directory to change the working directory entirely.

This setting has no effect when used in the uv pip interface.

-

May also be set with the UV_PROJECT environment variable.

--python-downloads-json-url python-downloads-json-url

URL pointing to JSON of custom Python installations.

-

Note that currently, only local paths are supported.

+

May also be set with the UV_PROJECT environment variable.

--python-downloads-json-url python-downloads-json-url

URL pointing to JSON of custom Python installations

--quiet, -q

Use quiet output.

Repeating this option, e.g., -qq, will enable a silent mode in which uv will write no output to stdout.

--script script

Find the environment for a Python script, rather than the current project

diff --git a/docs/reference/environment.md b/docs/reference/environment.md index 6b7f678bc..9bae80e46 100644 --- a/docs/reference/environment.md +++ b/docs/reference/environment.md @@ -537,10 +537,11 @@ Equivalent to the Managed Python installations information is hardcoded in the `uv` binary. -This variable can be set to a URL pointing to JSON to use as a list for Python installations. -This will allow for setting each property of the Python installation, mostly the url part for offline mirror. +This variable can be set to a local path or URL pointing to +a JSON list of Python installations to override the hardcoded list. -Note that currently, only local paths are supported. +This allows customizing the URLs for downloads or using slightly older or newer versions +of Python than the ones hardcoded into this build of `uv`. ### `UV_PYTHON_GRAALPY_BUILD` added in `0.8.14` diff --git a/docs/reference/settings.md b/docs/reference/settings.md index 2073d8244..db121b422 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -1933,8 +1933,6 @@ Whether to allow Python downloads. URL pointing to JSON of custom Python installations. -Note that currently, only local paths are supported. - **Default value**: `None` **Type**: `str` diff --git a/uv.schema.json b/uv.schema.json index b475bbd7a..7cd16427a 100644 --- a/uv.schema.json +++ b/uv.schema.json @@ -511,7 +511,7 @@ ] }, "python-downloads-json-url": { - "description": "URL pointing to JSON of custom Python installations.\n\nNote that currently, only local paths are supported.", + "description": "URL pointing to JSON of custom Python installations.", "type": [ "string", "null"