Python
/
uv
mirror of https://github.com/astral-sh/uv
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add `UV_PYTHON_DOWNLOADS_JSON_URL` to set custom managed python sources (#10939) 

## Summary

Add an option to overwrite the list of available Python downloads from a
local JSON file by using the environment variable
`UV_PYTHON_DOWNLOADS_JSON_URL`

as an experimental support for providing custom sources for Python
distribution binaries #8015

related #10203

I probably should make the JSON to be fetched from a remote URL instead
of a local file.
please let me know what you think and I will modify the code
accordingly.

## Test Plan

### normal run
```
root@75c66494ba8b:/# /code/target/release/uv python list
cpython-3.14.0a4+freethreaded-linux-x86_64-gnu    <download available>
cpython-3.14.0a4-linux-x86_64-gnu                 <download available>
cpython-3.13.1+freethreaded-linux-x86_64-gnu      <download available>
cpython-3.13.1-linux-x86_64-gnu                   <download available>
cpython-3.12.8-linux-x86_64-gnu                   <download available>
cpython-3.11.11-linux-x86_64-gnu                  <download available>
cpython-3.10.16-linux-x86_64-gnu                  <download available>
cpython-3.9.21-linux-x86_64-gnu                   <download available>
cpython-3.8.20-linux-x86_64-gnu                   <download available>
cpython-3.7.9-linux-x86_64-gnu                    <download available>
pypy-3.10.14-linux-x86_64-gnu                     <download available>
pypy-3.9.19-linux-x86_64-gnu                      <download available>
pypy-3.8.16-linux-x86_64-gnu                      <download available>
pypy-3.7.13-linux-x86_64-gnu                      <download available>
```

### empty JSON file
```sh
root@75c66494ba8b:/# export UV_PYTHON_DOWNLOADS_JSON_URL=/code/crates/uv-python/my-download-metadata.json 
root@75c66494ba8b:/# cat $UV_PYTHON_DOWNLOADS_JSON_URL 
{}
root@75c66494ba8b:/# /code/target/release/uv python list
root@75c66494ba8b:/# 
```

### JSON file with valid version
```sh
root@75c66494ba8b:/# export UV_PYTHON_DOWNLOADS_JSON_URL=/code/crates/uv-python/my-download-metadata.json 
root@75c66494ba8b:/# cat $UV_PYTHON_DOWNLOADS_JSON_URL 
{
  "cpython-3.11.9-linux-x86_64-gnu": {
    "name": "cpython",
    "arch": {
      "family": "x86_64",
      "variant": null
    },
    "os": "linux",
    "libc": "gnu",
    "major": 3,
    "minor": 11,
    "patch": 9,
    "prerelease": "",
    "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20240814/cpython-3.11.9%2B20240814-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz",
    "sha256": "daa487c7e73005c4426ac393273117cf0e2dc4ab9b2eeda366e04cd00eea00c9",
    "variant": null
  }
}
root@75c66494ba8b:/# /code/target/release/uv python list
cpython-3.11.9-linux-x86_64-gnu    <download available>
root@75c66494ba8b:/# 
```

### Remote Path

```sh
root@75c66494ba8b:/# export UV_PYTHON_DOWNLOADS_JSON_URL=http://a.com/file.json 
root@75c66494ba8b:/# /code/target/release/uv python list
error: Remote python downloads JSON is not yet supported, please use a local path (without `file://` prefix)
```

---------

Co-authored-by: Aria Desires <aria.desires@gmail.com>
This commit is contained in:
Meitar Reihan 2025-04-07 20:55:00 +03:00 committed by GitHub
parent c0ed5693a7
commit 2b62f73064
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 248 additions and 19303 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/sync-python-releases.yml vendored
View File

@ -24,7 +24,7 @@ jobs:
- name: Sync Python Releases
run: |
uv run -- fetch-download-metadata.py
uv run -- template-download-metadata.py
uv run -- minify-download-metadata.py
working-directory: ./crates/uv-python
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -35,7 +35,7 @@ jobs:
commit-message: "Sync latest Python releases"
add-paths: |
crates/uv-python/download-metadata.json
crates/uv-python/src/downloads.inc
crates/uv-python/src/download-metadata-minified.json
branch: "sync-python-releases"
title: "Sync latest Python releases"
body: "Automated update for Python releases."

1
Cargo.lock generated
View File

@ -5522,6 +5522,7 @@ dependencies = [
"indoc",
"insta",
"itertools 0.14.0",
"once_cell",
"owo-colors",
"procfs",
"regex",

1
Cargo.toml
View File

@ -125,6 +125,7 @@ memchr = { version = "2.7.4" }
miette = { version = "7.2.0", features = ["fancy-no-backtrace"] }
nanoid = { version = "0.4.0" }
nix = { version = "0.29.0" }
once_cell = { version = "1.20.2" }
owo-colors = { version = "4.1.0" }
path-slash = { version = "0.2.1" }
pathdiff = { version = "0.2.1" }

1
crates/uv-python/Cargo.toml
View File

@ -61,6 +61,7 @@ tokio-util = { workspace = true, features = ["compat"] }
tracing = { workspace = true }
url = { workspace = true }
which = { workspace = true }
once_cell = { workspace = true }
[target.'cfg(target_os = "linux")'.dependencies]
procfs = { workspace = true }

43
crates/uv-python/minify-download-metadata.py Executable file
View File

@ -0,0 +1,43 @@
# /// script
# requires-python = ">=3.12"
# ///
"""
Generate minified Python version download metadata json to embed in the binary.
Generates the `download-metadata-minified.json` file from the `download-metadata.json` file.
Usage:
uv run -- crates/uv-python/minify-download-metadata.py
"""
import json
from pathlib import Path
CRATE_ROOT = Path(__file__).parent
VERSION_METADATA = CRATE_ROOT / "download-metadata.json"
TARGET = CRATE_ROOT / "src" / "download-metadata-minified.json"
def process_json(data: dict) -> dict:
out_data = {}
for key, value in data.items():
# Exclude debug variants for now, we don't support them in the Rust side
if value["variant"] == "debug":
continue
out_data[key] = value
return out_data
def main() -> None:
json_data = json.loads(Path(VERSION_METADATA).read_text())
json_data = process_json(json_data)
json_string = json.dumps(json_data, separators=(",", ":"))
TARGET.write_text(json_string)
if __name__ == "__main__":
main()

1
crates/uv-python/src/download-metadata-minified.json Normal file
View File

File diff suppressed because one or more lines are too long

19073
crates/uv-python/src/downloads.inc
View File

File diff suppressed because it is too large Load Diff

39
crates/uv-python/src/downloads.inc.mustache
View File

@ -1,39 +0,0 @@
// Generated with `{{generated_with}}`
// From template at `{{generated_from}}`
use uv_pep440::{Prerelease, PrereleaseKind};
use crate::PythonVariant;
use crate::platform::ArchVariant;
pub(crate) const PYTHON_DOWNLOADS: &[ManagedPythonDownload] = &[
{{#versions}}
ManagedPythonDownload {
key: PythonInstallationKey {
major: {{value.major}},
minor: {{value.minor}},
patch: {{value.patch}},
prerelease: {{value.prerelease}},
implementation: LenientImplementationName::Known(ImplementationName::{{value.name}}),
arch: Arch{
family: target_lexicon::Architecture::{{value.arch_family}},
variant: {{value.arch_variant}},
},
os: Os(target_lexicon::OperatingSystem::{{value.os}}),
{{#value.libc}}
libc: Libc::Some(target_lexicon::Environment::{{.}}),
{{/value.libc}}
{{^value.libc}}
libc: Libc::None,
{{/value.libc}}
variant: {{value.variant}}
},
url: "{{value.url}}",
{{#value.sha256}}
sha256: Some("{{.}}")
{{/value.sha256}}
{{^value.sha256}}
sha256: None
{{/value.sha256}}
},
{{/versions}}
];

186
crates/uv-python/src/downloads.rs
View File

@ -1,3 +1,5 @@
use std::borrow::Cow;
use std::collections::HashMap;
use std::fmt::Display;
use std::io;
use std::path::{Path, PathBuf};
@ -7,8 +9,11 @@ use std::task::{Context, Poll};
use std::time::{Duration, SystemTime};
use futures::TryStreamExt;
use itertools::Itertools;
use once_cell::sync::OnceCell;
use owo_colors::OwoColorize;
use reqwest_retry::RetryPolicy;
use serde::Deserialize;
use thiserror::Error;
use tokio::io::{AsyncRead, ReadBuf};
use tokio_util::compat::FuturesAsyncReadCompatExt;
@ -30,6 +35,7 @@ use crate::installation::PythonInstallationKey;
use crate::libc::LibcDetectionError;
use crate::managed::ManagedPythonInstallation;
use crate::platform::{self, Arch, Libc, Os};
use crate::PythonVariant;
use crate::{Interpreter, PythonRequest, PythonVersion, VersionRequest};
#[derive(Error, Debug)]
@ -86,9 +92,13 @@ pub enum Error {
Mirror(&'static str, &'static str),
#[error(transparent)]
LibcDetection(#[from] LibcDetectionError),
#[error("Remote python downloads JSON is not yet supported, please use a local path (without `file://` prefix)")]
RemoteJSONNotSupported(),
#[error("The json of the python downloads is invalid: {0}")]
InvalidPythonDownloadsJSON(String, #[source] serde_json::Error),
}
#[derive(Debug, PartialEq)]
#[derive(Debug, PartialEq, Clone)]
pub struct ManagedPythonDownload {
key: PythonInstallationKey,
url: &'static str,
@ -245,9 +255,11 @@ impl PythonDownloadRequest {
}
/// Iterate over all [`PythonDownload`]'s that match this request.
pub fn iter_downloads(&self) -> impl Iterator<Item = &'static ManagedPythonDownload> + '_ {
ManagedPythonDownload::iter_all()
.filter(move |download| self.satisfied_by_download(download))
pub fn iter_downloads(
&self,
) -> Result<impl Iterator<Item = &'static ManagedPythonDownload> + use<'_>, Error> {
Ok(ManagedPythonDownload::iter_all()?
.filter(move |download| self.satisfied_by_download(download)))
}
/// Whether this request is satisfied by an installation key.
@ -445,7 +457,30 @@ impl FromStr for PythonDownloadRequest {
}
}
include!("downloads.inc");
const BUILTIN_PYTHON_DOWNLOADS_JSON: &str = include_str!("download-metadata-minified.json");
static PYTHON_DOWNLOADS: OnceCell<std::borrow::Cow<'static, [ManagedPythonDownload]>> =
OnceCell::new();
#[derive(Debug, Deserialize, Clone)]
struct JsonPythonDownload {
name: String,
arch: JsonArch,
os: String,
libc: String,
major: u8,
minor: u8,
patch: u8,
prerelease: Option<String>,
url: String,
sha256: Option<String>,
variant: Option<String>,
}
#[derive(Debug, Deserialize, Clone)]
struct JsonArch {
family: String,
variant: Option<String>,
}
#[derive(Debug, Clone)]
pub enum DownloadResult {
@ -459,14 +494,40 @@ impl ManagedPythonDownload {
request: &PythonDownloadRequest,
) -> Result<&'static ManagedPythonDownload, Error> {
request
.iter_downloads()
.iter_downloads()?
.next()
.ok_or(Error::NoDownloadFound(request.clone()))
}
/// Iterate over all [`ManagedPythonDownload`]s.
pub fn iter_all() -> impl Iterator<Item = &'static ManagedPythonDownload> {
PYTHON_DOWNLOADS.iter()
pub fn iter_all() -> Result<impl Iterator<Item = &'static ManagedPythonDownload>, Error> {
let runtime_source = std::env::var(EnvVars::UV_PYTHON_DOWNLOADS_JSON_URL);
let downloads = PYTHON_DOWNLOADS.get_or_try_init(|| {
let json_downloads: HashMap<String, JsonPythonDownload> =
if let Ok(json_source) = &runtime_source {
if Url::parse(json_source).is_ok() {
return Err(Error::RemoteJSONNotSupported());
}
let file = match fs_err::File::open(json_source) {
Ok(file) => file,
Err(e) => { Err(Error::Io(e)) }?,
};
serde_json::from_reader(file)
.map_err(|e| Error::InvalidPythonDownloadsJSON(json_source.clone(), e))?
} else {
serde_json::from_str(BUILTIN_PYTHON_DOWNLOADS_JSON).map_err(|e| {
Error::InvalidPythonDownloadsJSON("EMBEDDED IN THE BINARY".to_string(), e)
})?
};
let result = parse_json_downloads(json_downloads);
Ok(Cow::Owned(result))
})?;
Ok(downloads.iter())
}
pub fn url(&self) -> &'static str {
@ -702,6 +763,115 @@ impl ManagedPythonDownload {
}
}
fn parse_json_downloads(
json_downloads: HashMap<String, JsonPythonDownload>,
) -> Vec<ManagedPythonDownload> {
json_downloads
.into_iter()
.filter_map(|(key, entry)| {
let implementation = match entry.name.as_str() {
"cpython" => LenientImplementationName::Known(ImplementationName::CPython),
"pypy" => LenientImplementationName::Known(ImplementationName::PyPy),
_ => LenientImplementationName::Unknown(entry.name.clone()),
};
let arch_str = match entry.arch.family.as_str() {
"armv5tel" => "armv5te".to_string(),
// The `gc` variant of riscv64 is the common base instruction set and
// is the target in `python-build-standalone`
// See https://github.com/astral-sh/python-build-standalone/issues/504
"riscv64" => "riscv64gc".to_string(),
value => value.to_string(),
};
let arch_str = if let Some(variant) = entry.arch.variant {
format!("{arch_str}_{variant}")
} else {
arch_str
};
let arch = match Arch::from_str(&arch_str) {
Ok(arch) => arch,
Err(e) => {
debug!("Skipping entry {key}: Invalid arch '{arch_str}' - {e}");
return None;
}
};
let os = match Os::from_str(&entry.os) {
Ok(os) => os,
Err(e) => {
debug!("Skipping entry {}: Invalid OS '{}' - {}", key, entry.os, e);
return None;
}
};
let libc = match Libc::from_str(&entry.libc) {
Ok(libc) => libc,
Err(e) => {
debug!(
"Skipping entry {}: Invalid libc '{}' - {}",
key, entry.libc, e
);
return None;
}
};
let variant = match entry
.variant
.as_deref()
.map(PythonVariant::from_str)
.transpose()
{
Ok(Some(variant)) => variant,
Ok(None) => PythonVariant::default(),
Err(()) => {
debug!(
"Skipping entry {key}: Unknown python variant - {}",
entry.variant.unwrap_or_default()
);
return None;
}
};
let version_str = format!(
"{}.{}.{}{}",
entry.major,
entry.minor,
entry.patch,
entry.prerelease.as_deref().unwrap_or_default()
);
let version = match PythonVersion::from_str(&version_str) {
Ok(version) => version,
Err(e) => {
debug!("Skipping entry {key}: Invalid version '{version_str}' - {e}");
return None;
}
};
let url = Box::leak(entry.url.into_boxed_str()) as &'static str;
let sha256 = entry
.sha256
.map(|s| Box::leak(s.into_boxed_str()) as &'static str);
Some(ManagedPythonDownload {
key: PythonInstallationKey::new_from_version(
implementation,
&version,
os,
arch,
libc,
variant,
),
url,
sha256,
})
})
.sorted_by(|a, b| Ord::cmp(&b.key, &a.key))
.collect()
}
impl Error {
pub(crate) fn from_reqwest(url: Url, err: reqwest::Error) -> Self {
Self::NetworkError(url, WrappedReqwestError::from(err))

4
crates/uv-python/src/installation.rs
View File

@ -294,7 +294,7 @@ impl PythonInstallationKey {
}
}
fn new_from_version(
pub fn new_from_version(
implementation: LenientImplementationName,
version: &PythonVersion,
os: Os,
@ -482,6 +482,6 @@ impl Ord for PythonInstallationKey {
.then_with(|| self.os.to_string().cmp(&other.os.to_string()))
.then_with(|| self.arch.to_string().cmp(&other.arch.to_string()))
.then_with(|| self.libc.to_string().cmp(&other.libc.to_string()))
.then_with(|| self.variant.cmp(&other.variant))
.then_with(|| self.variant.cmp(&other.variant).reverse()) // we want Default to come first
}
}

178
crates/uv-python/template-download-metadata.py
View File

@ -1,178 +0,0 @@
# /// script
# requires-python = ">=3.12"
# dependencies = [
# "chevron-blue < 1",
# ]
# ///
"""
Generate static Rust code from Python version download metadata.
Generates the `downloads.inc` file from the `downloads.inc.mustache` template.
Usage:
uv run -- crates/uv-python/template-download-metadata.py
"""
import argparse
import json
import logging
import re
import subprocess
import sys
from pathlib import Path
from typing import Any
import chevron_blue
CRATE_ROOT = Path(__file__).parent
WORKSPACE_ROOT = CRATE_ROOT.parent.parent
VERSION_METADATA = CRATE_ROOT / "download-metadata.json"
TEMPLATE = CRATE_ROOT / "src" / "downloads.inc.mustache"
TARGET = TEMPLATE.with_suffix("")
PRERELEASE_PATTERN = re.compile(r"(a|b|rc)(\d+)")
def prepare_name(name: str) -> str:
match name:
case "cpython":
return "CPython"
case "pypy":
return "PyPy"
case _:
raise ValueError(f"Unknown implementation name: {name}")
def prepare_libc(libc: str) -> str | None:
if libc == "none":
return None
else:
return libc.title()
def prepare_variant(variant: str | None) -> str | None:
match variant:
case None:
return "PythonVariant::Default"
case "freethreaded":
return "PythonVariant::Freethreaded"
case "debug":
return "PythonVariant::Debug"
case _:
raise ValueError(f"Unknown variant: {variant}")
def prepare_arch(arch: dict) -> tuple[str, str]:
match arch["family"]:
# Special constructors
case "i686":
family = "X86_32(target_lexicon::X86_32Architecture::I686)"
case "aarch64":
family = "Aarch64(target_lexicon::Aarch64Architecture::Aarch64)"
case "armv5tel":
family = "Arm(target_lexicon::ArmArchitecture::Armv5te)"
case "armv7":
family = "Arm(target_lexicon::ArmArchitecture::Armv7)"
case "riscv64":
# The `gc` variant of riscv64 is the common base instruction set and
# is the target in `python-build-standalone`
# See https://github.com/astral-sh/python-build-standalone/issues/504
family = "Riscv64(target_lexicon::Riscv64Architecture::Riscv64gc)"
case value:
family = value.capitalize()
variant = (
f"Some(ArchVariant::{arch['variant'].capitalize()})"
if arch["variant"]
else "None"
)
return family, variant
def prepare_os(os: str) -> str:
match os:
# Special constructors
case "darwin":
return "Darwin(None)"
return os.title()
def prepare_prerelease(prerelease: str) -> str:
if not prerelease:
return "None"
if not (match := PRERELEASE_PATTERN.match(prerelease)):
raise ValueError(f"Invalid prerelease: {prerelease!r}")
kind, number = match.groups()
kind_mapping = {"a": "Alpha", "b": "Beta", "rc": "Rc"}
return f"Some(Prerelease {{ kind: PrereleaseKind::{kind_mapping[kind]}, number: {number} }})"
def prepare_value(value: dict) -> dict:
value["os"] = prepare_os(value["os"])
value["arch_family"], value["arch_variant"] = prepare_arch(value["arch"])
value["name"] = prepare_name(value["name"])
value["libc"] = prepare_libc(value["libc"])
value["prerelease"] = prepare_prerelease(value["prerelease"])
value["variant"] = prepare_variant(value["variant"])
return value
def main() -> None:
debug = logging.getLogger().getEffectiveLevel() <= logging.DEBUG
data: dict[str, Any] = {}
data["generated_with"] = Path(__file__).relative_to(WORKSPACE_ROOT).as_posix()
data["generated_from"] = TEMPLATE.relative_to(WORKSPACE_ROOT).as_posix()
data["versions"] = [
{"key": key, "value": prepare_value(value)}
for key, value in json.loads(VERSION_METADATA.read_text()).items()
# Exclude debug variants for now, we don't support them in the Rust side
if value["variant"] != "debug"
]
# Render the template
logging.info(f"Rendering `{TEMPLATE.name}`...")
output = chevron_blue.render(
template=TEMPLATE.read_text(), data=data, no_escape=True, warn=debug
)
# Update the file
logging.info(f"Updating `{TARGET}`...")
TARGET.write_text("// DO NOT EDIT\n//\n" + output)
subprocess.check_call(
["rustfmt", str(TARGET)],
stderr=subprocess.STDOUT,
stdout=sys.stderr if debug else subprocess.DEVNULL,
)
logging.info("Done!")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Generates Rust code for Python version metadata.",
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="Enable debug logging",
)
parser.add_argument(
"-q",
"--quiet",
action="store_true",
help="Disable logging",
)
args = parser.parse_args()
if args.quiet:
log_level = logging.CRITICAL
elif args.verbose:
log_level = logging.DEBUG
else:
log_level = logging.INFO
logging.basicConfig(level=log_level, format="%(message)s")
main()

8
crates/uv-static/src/env_vars.rs
View File

@ -257,6 +257,14 @@ impl EnvVars {
/// Specifies the directory for storing managed Python installations.
pub const UV_PYTHON_INSTALL_DIR: &'static str = "UV_PYTHON_INSTALL_DIR";
/// Managed Python installations information is hardcoded in the `uv` binary.
///
/// This variable can be set to a URL pointing to JSON to use as a list for Python installations.
/// This will allow for setting each property of the Python installation, mostly the url part for offline mirror.
///
/// Note that currently, only local paths are supported.
pub const UV_PYTHON_DOWNLOADS_JSON_URL: &'static str = "UV_PYTHON_DOWNLOADS_JSON_URL";
/// Managed Python installations are downloaded from the Astral
/// [`python-build-standalone`](https://github.com/astral-sh/python-build-standalone) project.
///

1
crates/uv/src/commands/python/list.rs
View File

@ -102,6 +102,7 @@ pub(crate) async fn list(
let downloads = download_request
.as_ref()
.map(PythonDownloadRequest::iter_downloads)
.transpose()?
.into_iter()
.flatten();

2
crates/uv/tests/it/python_install.rs
View File

@ -666,8 +666,8 @@ fn python_install_freethreaded() {
----- stderr -----
Searching for Python installations
Uninstalled 2 versions in [TIME]
- cpython-3.13.2-[PLATFORM]
- cpython-3.13.2+freethreaded-[PLATFORM] (python3.13t)
- cpython-3.13.2-[PLATFORM]
"###);
}

9
docs/configuration/environment.md
View File

@ -339,6 +339,15 @@ Equivalent to the
[`python-downloads`](../reference/settings.md#python-downloads) setting and, when disabled, the
`--no-python-downloads` option. Whether uv should allow Python downloads.
### `UV_PYTHON_DOWNLOADS_JSON_URL`
Managed Python installations information is hardcoded in the `uv` binary.
This variable can be set to a URL pointing to JSON to use as a list for Python installations.
This will allow for setting each property of the Python installation, mostly the url part for offline mirror.
Note that currently, only local paths are supported.
### `UV_PYTHON_INSTALL_DIR`
Specifies the directory for storing managed Python installations.