diff --git a/crates/uv-cli/src/lib.rs b/crates/uv-cli/src/lib.rs index 10f3ec846..2a87a0d45 100644 --- a/crates/uv-cli/src/lib.rs +++ b/crates/uv-cli/src/lib.rs @@ -6113,6 +6113,36 @@ pub struct PythonDirArgs { pub bin: bool, } +#[derive(Args)] +pub struct PythonInstallCompileBytecodeArgs { + /// Compile Python's standard library to bytecode after installation. + /// + /// By default, uv does not compile Python (`.py`) files to bytecode (`__pycache__/*.pyc`); + /// instead, compilation is performed lazily the first time a module is imported. For use-cases + /// in which start time is important, such as CLI applications and Docker containers, this + /// option can be enabled to trade longer installation times and some additional disk space for + /// faster start times. + /// + /// When enabled, uv will process the Python version's `stdlib` directory. It will ignore any + /// compilation errors. + #[arg( + long, + alias = "compile", + overrides_with("no_compile_bytecode"), + env = EnvVars::UV_COMPILE_BYTECODE, + value_parser = clap::builder::BoolishValueParser::new(), + )] + pub compile_bytecode: bool, + + #[arg( + long, + alias = "no-compile", + overrides_with("compile_bytecode"), + hide = true + )] + pub no_compile_bytecode: bool, +} + #[derive(Args)] pub struct PythonInstallArgs { /// The directory to store the Python installation in. @@ -6232,6 +6262,9 @@ pub struct PythonInstallArgs { /// If multiple Python versions are requested, uv will exit with an error. #[arg(long, conflicts_with("no_bin"))] pub default: bool, + + #[command(flatten)] + pub compile_bytecode: PythonInstallCompileBytecodeArgs, } impl PythonInstallArgs { @@ -6292,6 +6325,9 @@ pub struct PythonUpgradeArgs { /// URL pointing to JSON of custom Python installations. #[arg(long, value_hint = ValueHint::Other)] pub python_downloads_json_url: Option, + + #[command(flatten)] + pub compile_bytecode: PythonInstallCompileBytecodeArgs, } impl PythonUpgradeArgs { diff --git a/crates/uv/src/commands/python/install.rs b/crates/uv/src/commands/python/install.rs index 480731cbe..abb530384 100644 --- a/crates/uv/src/commands/python/install.rs +++ b/crates/uv/src/commands/python/install.rs @@ -5,16 +5,18 @@ use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::str::FromStr; -use anyhow::{Error, Result}; -use futures::StreamExt; -use futures::stream::FuturesUnordered; +use anyhow::{Context, Error, Result}; +use futures::{StreamExt, join}; use indexmap::IndexSet; use itertools::{Either, Itertools}; use owo_colors::{AnsiColors, OwoColorize}; use rustc_hash::{FxHashMap, FxHashSet}; -use tracing::{debug, trace}; +use tokio::sync::mpsc; +use tracing::{debug, trace, warn}; +use uv_cache::Cache; use uv_client::BaseClientBuilder; +use uv_configuration::Concurrency; use uv_fs::Simplified; use uv_platform::{Arch, Libc}; use uv_preview::{Preview, PreviewFeatures}; @@ -27,8 +29,9 @@ use uv_python::managed::{ create_link_to_executable, python_executable_dir, }; use uv_python::{ - PythonDownloads, PythonInstallationKey, PythonInstallationMinorVersionKey, PythonRequest, - PythonVersionFile, VersionFileDiscoveryOptions, VersionFilePreference, VersionRequest, + ImplementationName, Interpreter, PythonDownloads, PythonInstallationKey, + PythonInstallationMinorVersionKey, PythonRequest, PythonVersionFile, + VersionFileDiscoveryOptions, VersionFilePreference, VersionRequest, }; use uv_shell::Shell; use uv_trampoline_builder::{Launcher, LauncherKind}; @@ -191,6 +194,114 @@ pub(crate) async fn install( default: bool, python_downloads: PythonDownloads, no_config: bool, + compile_bytecode: bool, + concurrency: &Concurrency, + cache: &Cache, + preview: Preview, + printer: Printer, +) -> Result { + let (sender, mut receiver) = mpsc::unbounded_channel(); + let compiler = async { + let mut total_files = 0; + let mut total_elapsed = std::time::Duration::default(); + let mut total_skipped = 0; + while let Some(installation) = receiver.recv().await { + if let Some((files, elapsed)) = + compile_stdlib_bytecode(&installation, concurrency, cache) + .await + .with_context(|| { + format!( + "Failed to bytecode-compile Python standard library for: {}", + installation.key() + ) + })? + { + total_files += files; + total_elapsed += elapsed; + } else { + total_skipped += 1; + } + } + Ok::<_, anyhow::Error>((total_files, total_elapsed, total_skipped)) + }; + + let installer = perform_install( + project_dir, + install_dir, + targets, + reinstall, + upgrade, + bin, + registry, + force, + python_install_mirror, + pypy_install_mirror, + python_downloads_json_url, + client_builder, + default, + python_downloads, + no_config, + compile_bytecode.then_some(sender), + concurrency, + preview, + printer, + ); + + let (installer_result, compiler_result) = join!(installer, compiler); + + let (total_files, total_elapsed, total_skipped) = compiler_result?; + if total_files > 0 { + let s = if total_files == 1 { "" } else { "s" }; + writeln!( + printer.stderr(), + "{}", + format!( + "Bytecode compiled {} {}{}", + format!("{total_files} file{s}").bold(), + format!("in {}", elapsed(total_elapsed)).dimmed(), + if total_skipped > 0 { + format!( + " (skipped {total_skipped} incompatible version{})", + if total_skipped == 1 { "" } else { "s" } + ) + } else { + String::new() + } + .dimmed() + ) + .dimmed() + )?; + } else if total_skipped > 0 { + writeln!( + printer.stderr(), + "{}", + format!("No compatible versions to bytecode compile (skipped {total_skipped})") + .dimmed() + )?; + } + + installer_result +} + +#[allow(clippy::fn_params_excessive_bools)] +async fn perform_install( + project_dir: &Path, + install_dir: Option, + targets: Vec, + reinstall: bool, + upgrade: PythonUpgrade, + bin: Option, + registry: Option, + force: bool, + python_install_mirror: Option, + pypy_install_mirror: Option, + python_downloads_json_url: Option, + client_builder: BaseClientBuilder<'_>, + default: bool, + python_downloads: PythonDownloads, + no_config: bool, + bytecode_compilation_sender: Option>, + concurrency: &Concurrency, preview: Preview, printer: Printer, ) -> Result { @@ -433,6 +544,16 @@ pub(crate) async fn install( }) }; + // For all satisfied installs, bytecode compile them now before any future + // early return. + if let Some(ref sender) = bytecode_compilation_sender { + satisfied + .iter() + .copied() + .cloned() + .try_for_each(|installation| sender.send(installation))?; + } + // Check if Python downloads are banned if matches!(python_downloads, PythonDownloads::Never) && !unsatisfied.is_empty() { writeln!( @@ -458,10 +579,9 @@ pub(crate) async fn install( // Download and unpack the Python versions concurrently let reporter = PythonDownloadReporter::new(printer, Some(downloads.len() as u64)); - let mut tasks = FuturesUnordered::new(); - for download in &downloads { - tasks.push(async { + let mut tasks = futures::stream::iter(&downloads) + .map(async |download| { ( *download, download @@ -477,8 +597,8 @@ pub(crate) async fn install( ) .await, ) - }); - } + }) + .buffer_unordered(concurrency.downloads); let mut errors = vec![]; let mut downloaded = Vec::with_capacity(downloads.len()); @@ -493,6 +613,9 @@ pub(crate) async fn install( }; let installation = ManagedPythonInstallation::new(path, download); + if let Some(ref sender) = bytecode_compilation_sender { + sender.send(installation.clone())?; + } changelog.installed.insert(installation.key().clone()); for request in &requests { // Take note of which installations satisfied which requests @@ -1071,6 +1194,53 @@ fn create_bin_links( } } +/// Attempt to compile the bytecode for a [`ManagedPythonInstallation`]'s stdlib +async fn compile_stdlib_bytecode( + installation: &ManagedPythonInstallation, + concurrency: &Concurrency, + cache: &Cache, +) -> Result> { + let start = std::time::Instant::now(); + + // Explicit matching so this heuristic is updated for future additions + match installation.implementation() { + ImplementationName::Pyodide => return Ok(None), + ImplementationName::GraalPy | ImplementationName::PyPy | ImplementationName::CPython => (), + } + + let interpreter = Interpreter::query(installation.executable(false), cache) + .context("Couldn't locate the interpreter")?; + + // Ensure the bytecode compilation occurs in the correct place, in case the installed + // interpreter reports a weird stdlib path. + let interpreter_path = installation.path().canonicalize()?; + let stdlib_path = match interpreter.stdlib().canonicalize() { + Ok(path) if path.starts_with(&interpreter_path) => path, + _ => { + warn!( + "The stdlib path for {} ({}) is not a subdirectory of its installation path ({}).", + installation.key(), + interpreter.stdlib().display(), + interpreter_path.display() + ); + return Ok(None); + } + }; + + let files = uv_installer::compile_tree( + &stdlib_path, + &installation.executable(false), + concurrency, + cache.root(), + ) + .await + .with_context(|| format!("Error compiling bytecode in: {}", stdlib_path.display()))?; + if files == 0 { + return Ok(None); + } + Ok(Some((files, start.elapsed()))) +} + pub(crate) fn format_executables( event: &ChangeEvent, executables: &FxHashMap>, diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index d83ff4055..74eaf3863 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -1645,6 +1645,9 @@ async fn run(mut cli: Cli) -> Result { let args = settings::PythonInstallSettings::resolve(args, filesystem, environment); show_settings!(args); + // Initialize the cache. + let cache = cache.init().await?; + commands::python_install( &project_dir, args.install_dir, @@ -1661,6 +1664,9 @@ async fn run(mut cli: Cli) -> Result { args.default, globals.python_downloads, cli.top_level.no_config, + args.compile_bytecode, + &globals.concurrency, + &cache, globals.preview, printer, ) @@ -1674,6 +1680,9 @@ async fn run(mut cli: Cli) -> Result { show_settings!(args); let upgrade = commands::PythonUpgrade::Enabled(commands::PythonUpgradeSource::Upgrade); + // Initialize the cache. + let cache = cache.init().await?; + commands::python_install( &project_dir, args.install_dir, @@ -1690,6 +1699,9 @@ async fn run(mut cli: Cli) -> Result { args.default, globals.python_downloads, cli.top_level.no_config, + args.compile_bytecode, + &globals.concurrency, + &cache, globals.preview, printer, ) diff --git a/crates/uv/src/settings.rs b/crates/uv/src/settings.rs index f0b169b20..34a1b0e74 100644 --- a/crates/uv/src/settings.rs +++ b/crates/uv/src/settings.rs @@ -1280,6 +1280,7 @@ pub(crate) struct PythonInstallSettings { pub(crate) pypy_install_mirror: Option, pub(crate) python_downloads_json_url: Option, pub(crate) default: bool, + pub(crate) compile_bytecode: bool, } impl PythonInstallSettings { @@ -1319,6 +1320,7 @@ impl PythonInstallSettings { pypy_mirror: _, python_downloads_json_url: _, default, + compile_bytecode, } = args; Self { @@ -1338,6 +1340,12 @@ impl PythonInstallSettings { pypy_install_mirror, python_downloads_json_url, default, + compile_bytecode: flag( + compile_bytecode.compile_bytecode, + compile_bytecode.no_compile_bytecode, + "compile-bytecode", + ) + .unwrap_or_default(), } } } @@ -1356,6 +1364,7 @@ pub(crate) struct PythonUpgradeSettings { pub(crate) python_downloads_json_url: Option, pub(crate) default: bool, pub(crate) bin: Option, + pub(crate) compile_bytecode: bool, } impl PythonUpgradeSettings { @@ -1393,6 +1402,7 @@ impl PythonUpgradeSettings { pypy_mirror: _, reinstall, python_downloads_json_url: _, + compile_bytecode, } = args; Self { @@ -1406,6 +1416,12 @@ impl PythonUpgradeSettings { python_downloads_json_url, default, bin, + compile_bytecode: flag( + compile_bytecode.compile_bytecode, + compile_bytecode.no_compile_bytecode, + "compile-bytecode", + ) + .unwrap_or_default(), } } } diff --git a/crates/uv/tests/it/common/mod.rs b/crates/uv/tests/it/common/mod.rs index 46ea71091..242ead7b4 100644 --- a/crates/uv/tests/it/common/mod.rs +++ b/crates/uv/tests/it/common/mod.rs @@ -481,6 +481,16 @@ impl TestContext { self } + /// Add a filter for (bytecode) compilation file counts + #[must_use] + pub fn with_filtered_compiled_file_count(mut self) -> Self { + self.filters.push(( + r"compiled \d+ files".to_string(), + "compiled [COUNT] files".to_string(), + )); + self + } + /// Adds filters for non-deterministic `CycloneDX` data pub fn with_cyclonedx_filters(mut self) -> Self { self.filters.push(( diff --git a/crates/uv/tests/it/help.rs b/crates/uv/tests/it/help.rs index c6e7144b0..d1d09cc4b 100644 --- a/crates/uv/tests/it/help.rs +++ b/crates/uv/tests/it/help.rs @@ -576,6 +576,20 @@ fn help_subsubcommand() { If multiple Python versions are requested, uv will exit with an error. + --compile-bytecode + Compile Python's standard library to bytecode after installation. + + By default, uv does not compile Python (`.py`) files to bytecode (`__pycache__/*.pyc`); + instead, compilation is performed lazily the first time a module is imported. For + use-cases in which start time is important, such as CLI applications and Docker + containers, this option can be enabled to trade longer installation times and some + additional disk space for faster start times. + + When enabled, uv will process the Python version's `stdlib` directory. It will ignore any + compilation errors. + + [env: UV_COMPILE_BYTECODE=] + Cache options: -n, --no-cache Avoid reading from or writing to the cache, instead using a temporary directory for the @@ -827,6 +841,9 @@ fn help_flag_subsubcommand() { Upgrade existing Python installations to the latest patch version --default Use as the default Python version + --compile-bytecode + Compile Python's standard library to bytecode after installation [env: + UV_COMPILE_BYTECODE=] Cache options: -n, --no-cache Avoid reading from or writing to the cache, instead using a temporary diff --git a/crates/uv/tests/it/python_install.rs b/crates/uv/tests/it/python_install.rs index 2d8cbc6a1..37248392c 100644 --- a/crates/uv/tests/it/python_install.rs +++ b/crates/uv/tests/it/python_install.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use std::{env, path::Path, process::Command}; use crate::common::{TestContext, uv_snapshot}; +use anyhow::Context; use assert_cmd::assert::OutputAssertExt; use assert_fs::{ assert::PathAssert, @@ -15,6 +16,7 @@ use tracing::debug; use uv_fs::Simplified; use uv_static::EnvVars; +use walkdir::WalkDir; #[test] fn python_install() { @@ -3951,3 +3953,253 @@ fn python_install_upgrade_version_file() { hint: The version request came from a `.python-version` file; change the patch version in the file to upgrade instead "); } + +#[test] +fn python_install_compile_bytecode() -> anyhow::Result<()> { + fn count_files_by_ext(dir: &Path, extension: &str) -> anyhow::Result { + let mut count = 0; + let walker = WalkDir::new(dir).into_iter(); + for entry in walker { + let entry = entry?; + let path = entry.path(); + if entry.metadata()?.is_file() && path.extension().is_some_and(|ext| ext == extension) { + count += 1; + } + } + Ok(count) + } + + let context: TestContext = TestContext::new_with_versions(&[]) + .with_filtered_python_keys() + .with_filtered_exe_suffix() + .with_filtered_compiled_file_count() + .with_managed_python_dirs() + .with_empty_python_install_mirror() + .with_python_download_cache(); + + // Install 3.14 and compile its bytecode + uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("3.14"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Installed Python 3.14.2 in [TIME] + + cpython-3.14.2-[PLATFORM] (python3.14) + Bytecode compiled [COUNT] files in [TIME] + "); + + // Find the stdlib path for cpython 3.14 + let bin_path = context + .bin_dir + .child(format!("python3.14{}", std::env::consts::EXE_SUFFIX)); + + #[cfg(unix)] + let stdlib = fs_err::read_link(bin_path)? + .parent() + .context("Python binary should be a child of `bin`")? + .parent() + .context("`bin` directory should be a child of the installation path")? + .join("lib") + .join("python3.14"); + #[cfg(windows)] + let stdlib = launcher_path(&bin_path) + .parent() + .context("Python binary should be a child of the installation path")? + .join("Lib"); + + // And the count should match + let pyc_count = count_files_by_ext(&stdlib, "pyc")?; + let py_count = count_files_by_ext(&stdlib, "py")?; + assert_eq!(pyc_count, py_count); + + // Attempting to install with --compile-bytecode should (currently) + // unconditionally re-run the bytecode compiler + uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("3.14"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Python 3.14 is already installed + Bytecode compiled [COUNT] files in [TIME] + "); + + // Reinstalling with --compile-bytecode should compile bytecode. + uv_snapshot!(context.filters(), context.python_install().arg("--reinstall").arg("--compile-bytecode").arg("3.14"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Installed Python 3.14.2 in [TIME] + ~ cpython-3.14.2-[PLATFORM] (python3.14) + Bytecode compiled [COUNT] files in [TIME] + "); + + Ok(()) +} + +#[test] +fn python_install_compile_bytecode_existing() { + let context: TestContext = TestContext::new_with_versions(&[]) + .with_filtered_python_keys() + .with_filtered_exe_suffix() + .with_filtered_compiled_file_count() + .with_managed_python_dirs() + .with_empty_python_install_mirror() + .with_python_download_cache(); + + // A fresh install should be able to be compiled later + uv_snapshot!(context.filters(), context.python_install().arg("3.14"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Installed Python 3.14.2 in [TIME] + + cpython-3.14.2-[PLATFORM] (python3.14) + "); + + uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("3.14"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Python 3.14 is already installed + Bytecode compiled [COUNT] files in [TIME] + "); +} + +#[test] +fn python_install_compile_bytecode_upgrade() { + let context: TestContext = TestContext::new_with_versions(&[]) + .with_filtered_python_keys() + .with_filtered_exe_suffix() + .with_filtered_compiled_file_count() + .with_managed_python_dirs() + .with_empty_python_install_mirror() + .with_python_download_cache(); + + // An upgrade should also compile bytecode + uv_snapshot!(context.filters(), context.python_install().arg("3.14.0"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Installed Python 3.14.0 in [TIME] + + cpython-3.14.0-[PLATFORM] (python3.14) + "); + + uv_snapshot!(context.filters(), context.python_install().arg("--upgrade").arg("--compile-bytecode").arg("3.14"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Installed Python 3.14.2 in [TIME] + + cpython-3.14.2-[PLATFORM] (python3.14) + Bytecode compiled [COUNT] files in [TIME] + "); +} + +#[test] +fn python_install_compile_bytecode_multiple() { + let context: TestContext = TestContext::new_with_versions(&[]) + .with_filtered_python_keys() + .with_filtered_exe_suffix() + .with_filtered_compiled_file_count() + .with_managed_python_dirs() + .with_empty_python_install_mirror() + .with_python_download_cache(); + + // Should handle installing and compiling multiple versions correctly + uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("3.14").arg("3.12"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Installed 2 versions in [TIME] + + cpython-3.12.12-[PLATFORM] (python3.12) + + cpython-3.14.2-[PLATFORM] (python3.14) + Bytecode compiled [COUNT] files in [TIME] + "); +} + +#[cfg(unix)] // Pyodide cannot be used on Windows +#[test] +fn python_install_compile_bytecode_pyodide() { + let context: TestContext = TestContext::new_with_versions(&[]) + .with_filtered_python_keys() + .with_filtered_exe_suffix() + .with_filtered_compiled_file_count() + .with_managed_python_dirs() + .with_empty_python_install_mirror() + .with_python_download_cache(); + + // Should warn on explicit pyodide installation + uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("cpython-3.13.2-emscripten-wasm32-musl"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Installed Python 3.13.2 in [TIME] + + pyodide-3.13.2-emscripten-wasm32-musl (python3.13) + No compatible versions to bytecode compile (skipped 1) + "); + + // TODO(tk) There's a bug with python_upgrade when pyodide is installed which leads to + // `error: No download found for request: pyodide-3.13-emscripten-wasm32-musl` + //// Recompilation where pyodide isn't explicitly specified shouldn't warn + //uv_snapshot!(context.filters(), context.python_upgrade().arg("--compile-bytecode"), @r"TODO"); +} + +#[test] +fn python_install_compile_bytecode_graalpy() { + let context: TestContext = TestContext::new_with_versions(&[]) + .with_filtered_python_keys() + .with_filtered_exe_suffix() + .with_filtered_compiled_file_count() + .with_managed_python_dirs() + .with_empty_python_install_mirror() + .with_python_download_cache(); + + // Should work for graalpy + uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("graalpy-3.12"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Installed Python 3.12.0 in [TIME] + + graalpy-3.12.0-[PLATFORM] (python3.12) + Bytecode compiled [COUNT] files in [TIME] + "); +} + +#[test] +fn python_install_compile_bytecode_pypy() { + let context: TestContext = TestContext::new_with_versions(&[]) + .with_filtered_python_keys() + .with_filtered_exe_suffix() + .with_filtered_compiled_file_count() + .with_managed_python_dirs() + .with_empty_python_install_mirror() + .with_python_download_cache(); + + // Should work for pypy + uv_snapshot!(context.filters(), context.python_install().arg("--compile-bytecode").arg("pypy-3.11"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Installed Python 3.11.13 in [TIME] + + pypy-3.11.13-[PLATFORM] (python3.11) + Bytecode compiled [COUNT] files in [TIME] + "); +} diff --git a/docs/guides/integration/docker.md b/docs/guides/integration/docker.md index 80dddba65..8c1ccc485 100644 --- a/docs/guides/integration/docker.md +++ b/docs/guides/integration/docker.md @@ -326,11 +326,12 @@ See a complete example in the ### Compiling bytecode Compiling Python source files to bytecode is typically desirable for production images as it tends -to improve startup time (at the cost of increased installation time). +to improve startup time (at the cost of increased installation time and image size). To enable bytecode compilation, use the `--compile-bytecode` flag: ```dockerfile title="Dockerfile" +RUN uv python install --compile-bytecode RUN uv sync --compile-bytecode ``` @@ -341,6 +342,13 @@ commands within the Dockerfile compile bytecode: ENV UV_COMPILE_BYTECODE=1 ``` +!!! note + + uv will only compile the standard library of _managed_ Python versions during + `uv python install`. The distributor of unmanaged Python versions decides if the + standard library is pre-compiled. For example, the official `python` image will not + have a compiled standard library. + ### Caching A [cache mount](https://docs.docker.com/build/guide/mounts/#add-a-cache-mount) can be used to