Use install concurrency for bytecode compilation too (#11615)

Instead of always using all available threads for bytecode compilation,
respect `UV_CONCURRENT_INSTALLS`, so the parallelism is configurable
instead of hardcoded. We reuse the install limit since bytecode
compilation only runs after install.
This commit is contained in:
konsti 2025-02-20 12:23:40 +01:00 committed by GitHub
parent ab551eaf34
commit f9b638a296
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 25 additions and 16 deletions

1
Cargo.lock generated
View File

@ -4928,6 +4928,7 @@ dependencies = [
"uv-cache", "uv-cache",
"uv-cli", "uv-cli",
"uv-client", "uv-client",
"uv-configuration",
"uv-distribution-filename", "uv-distribution-filename",
"uv-distribution-types", "uv-distribution-types",
"uv-extract", "uv-extract",

View File

@ -19,6 +19,7 @@ workspace = true
uv-cache = { workspace = true, features = ["clap"] } uv-cache = { workspace = true, features = ["clap"] }
uv-cli = { workspace = true } uv-cli = { workspace = true }
uv-client = { workspace = true } uv-client = { workspace = true }
uv-configuration = { workspace = true }
uv-distribution-filename = { workspace = true } uv-distribution-filename = { workspace = true }
uv-distribution-types = { workspace = true } uv-distribution-types = { workspace = true }
uv-extract = { workspace = true, optional = true } uv-extract = { workspace = true, optional = true }

View File

@ -4,6 +4,7 @@ use clap::Parser;
use tracing::info; use tracing::info;
use uv_cache::{Cache, CacheArgs}; use uv_cache::{Cache, CacheArgs};
use uv_configuration::Concurrency;
use uv_python::{EnvironmentPreference, PythonEnvironment, PythonRequest}; use uv_python::{EnvironmentPreference, PythonEnvironment, PythonRequest};
#[derive(Parser)] #[derive(Parser)]
@ -33,6 +34,7 @@ pub(crate) async fn compile(args: CompileArgs) -> anyhow::Result<()> {
let files = uv_installer::compile_tree( let files = uv_installer::compile_tree(
&fs_err::canonicalize(args.root)?, &fs_err::canonicalize(args.root)?,
&interpreter, &interpreter,
&Concurrency::default(),
cache.root(), cache.root(),
) )
.await?; .await?;

View File

@ -1,4 +1,3 @@
use std::num::NonZeroUsize;
use std::panic::AssertUnwindSafe; use std::panic::AssertUnwindSafe;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::process::Stdio; use std::process::Stdio;
@ -14,6 +13,7 @@ use tokio::sync::oneshot;
use tracing::{debug, instrument}; use tracing::{debug, instrument};
use walkdir::WalkDir; use walkdir::WalkDir;
use uv_configuration::Concurrency;
use uv_fs::Simplified; use uv_fs::Simplified;
use uv_static::EnvVars; use uv_static::EnvVars;
use uv_warnings::warn_user; use uv_warnings::warn_user;
@ -71,6 +71,7 @@ pub enum CompileError {
pub async fn compile_tree( pub async fn compile_tree(
dir: &Path, dir: &Path,
python_executable: &Path, python_executable: &Path,
concurrency: &Concurrency,
cache: &Path, cache: &Path,
) -> Result<usize, CompileError> { ) -> Result<usize, CompileError> {
debug_assert!( debug_assert!(
@ -78,13 +79,10 @@ pub async fn compile_tree(
"compileall doesn't work with relative paths: `{}`", "compileall doesn't work with relative paths: `{}`",
dir.display() dir.display()
); );
let worker_count = std::thread::available_parallelism().unwrap_or_else(|err| { let worker_count = concurrency.installs;
warn_user!("Couldn't determine number of cores, compiling with a single thread: {err}");
NonZeroUsize::MIN
});
// A larger buffer is significantly faster than just 1 or the worker count. // A larger buffer is significantly faster than just 1 or the worker count.
let (sender, receiver) = async_channel::bounded::<PathBuf>(worker_count.get() * 10); let (sender, receiver) = async_channel::bounded::<PathBuf>(worker_count * 10);
// Running Python with an actual file will produce better error messages. // Running Python with an actual file will produce better error messages.
let tempdir = tempdir_in(cache).map_err(CompileError::TempFile)?; let tempdir = tempdir_in(cache).map_err(CompileError::TempFile)?;
@ -92,7 +90,7 @@ pub async fn compile_tree(
debug!("Starting {} bytecode compilation workers", worker_count); debug!("Starting {} bytecode compilation workers", worker_count);
let mut worker_handles = Vec::new(); let mut worker_handles = Vec::new();
for _ in 0..worker_count.get() { for _ in 0..worker_count {
let (tx, rx) = oneshot::channel(); let (tx, rx) = oneshot::channel();
let worker = worker( let worker = worker(

View File

@ -47,6 +47,7 @@ pub(crate) use tool::uninstall::uninstall as tool_uninstall;
pub(crate) use tool::update_shell::update_shell as tool_update_shell; pub(crate) use tool::update_shell::update_shell as tool_update_shell;
pub(crate) use tool::upgrade::upgrade as tool_upgrade; pub(crate) use tool::upgrade::upgrade as tool_upgrade;
use uv_cache::Cache; use uv_cache::Cache;
use uv_configuration::Concurrency;
use uv_distribution_types::InstalledMetadata; use uv_distribution_types::InstalledMetadata;
use uv_fs::{Simplified, CWD}; use uv_fs::{Simplified, CWD};
use uv_installer::compile_tree; use uv_installer::compile_tree;
@ -147,6 +148,7 @@ pub(super) struct DryRunEvent<T: Display> {
/// See the `--compile` option on `pip sync` and `pip install`. /// See the `--compile` option on `pip sync` and `pip install`.
pub(super) async fn compile_bytecode( pub(super) async fn compile_bytecode(
venv: &PythonEnvironment, venv: &PythonEnvironment,
concurrency: &Concurrency,
cache: &Cache, cache: &Cache,
printer: Printer, printer: Printer,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
@ -154,14 +156,19 @@ pub(super) async fn compile_bytecode(
let mut files = 0; let mut files = 0;
for site_packages in venv.site_packages() { for site_packages in venv.site_packages() {
let site_packages = CWD.join(site_packages); let site_packages = CWD.join(site_packages);
files += compile_tree(&site_packages, venv.python_executable(), cache.root()) files += compile_tree(
.await &site_packages,
.with_context(|| { venv.python_executable(),
format!( concurrency,
"Failed to bytecode-compile Python file in: {}", cache.root(),
site_packages.user_display() )
) .await
})?; .with_context(|| {
format!(
"Failed to bytecode-compile Python file in: {}",
site_packages.user_display()
)
})?;
} }
let s = if files == 1 { "" } else { "s" }; let s = if files == 1 { "" } else { "s" };
writeln!( writeln!(

View File

@ -554,7 +554,7 @@ pub(crate) async fn install(
} }
if compile { if compile {
compile_bytecode(venv, cache, printer).await?; compile_bytecode(venv, &concurrency, cache, printer).await?;
} }
// Construct a summary of the changes made to the environment. // Construct a summary of the changes made to the environment.