Use install concurrency for bytecode compilation too (#11615)

Instead of always using all available threads for bytecode compilation,
respect `UV_CONCURRENT_INSTALLS`, so the parallelism is configurable
instead of hardcoded. We reuse the install limit since bytecode
compilation only runs after install.
This commit is contained in:
konsti 2025-02-20 12:23:40 +01:00 committed by GitHub
parent ab551eaf34
commit f9b638a296
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 25 additions and 16 deletions

1
Cargo.lock generated
View File

@ -4928,6 +4928,7 @@ dependencies = [
"uv-cache",
"uv-cli",
"uv-client",
"uv-configuration",
"uv-distribution-filename",
"uv-distribution-types",
"uv-extract",

View File

@ -19,6 +19,7 @@ workspace = true
uv-cache = { workspace = true, features = ["clap"] }
uv-cli = { workspace = true }
uv-client = { workspace = true }
uv-configuration = { workspace = true }
uv-distribution-filename = { workspace = true }
uv-distribution-types = { workspace = true }
uv-extract = { workspace = true, optional = true }

View File

@ -4,6 +4,7 @@ use clap::Parser;
use tracing::info;
use uv_cache::{Cache, CacheArgs};
use uv_configuration::Concurrency;
use uv_python::{EnvironmentPreference, PythonEnvironment, PythonRequest};
#[derive(Parser)]
@ -33,6 +34,7 @@ pub(crate) async fn compile(args: CompileArgs) -> anyhow::Result<()> {
let files = uv_installer::compile_tree(
&fs_err::canonicalize(args.root)?,
&interpreter,
&Concurrency::default(),
cache.root(),
)
.await?;

View File

@ -1,4 +1,3 @@
use std::num::NonZeroUsize;
use std::panic::AssertUnwindSafe;
use std::path::{Path, PathBuf};
use std::process::Stdio;
@ -14,6 +13,7 @@ use tokio::sync::oneshot;
use tracing::{debug, instrument};
use walkdir::WalkDir;
use uv_configuration::Concurrency;
use uv_fs::Simplified;
use uv_static::EnvVars;
use uv_warnings::warn_user;
@ -71,6 +71,7 @@ pub enum CompileError {
pub async fn compile_tree(
dir: &Path,
python_executable: &Path,
concurrency: &Concurrency,
cache: &Path,
) -> Result<usize, CompileError> {
debug_assert!(
@ -78,13 +79,10 @@ pub async fn compile_tree(
"compileall doesn't work with relative paths: `{}`",
dir.display()
);
let worker_count = std::thread::available_parallelism().unwrap_or_else(|err| {
warn_user!("Couldn't determine number of cores, compiling with a single thread: {err}");
NonZeroUsize::MIN
});
let worker_count = concurrency.installs;
// A larger buffer is significantly faster than just 1 or the worker count.
let (sender, receiver) = async_channel::bounded::<PathBuf>(worker_count.get() * 10);
let (sender, receiver) = async_channel::bounded::<PathBuf>(worker_count * 10);
// Running Python with an actual file will produce better error messages.
let tempdir = tempdir_in(cache).map_err(CompileError::TempFile)?;
@ -92,7 +90,7 @@ pub async fn compile_tree(
debug!("Starting {} bytecode compilation workers", worker_count);
let mut worker_handles = Vec::new();
for _ in 0..worker_count.get() {
for _ in 0..worker_count {
let (tx, rx) = oneshot::channel();
let worker = worker(

View File

@ -47,6 +47,7 @@ pub(crate) use tool::uninstall::uninstall as tool_uninstall;
pub(crate) use tool::update_shell::update_shell as tool_update_shell;
pub(crate) use tool::upgrade::upgrade as tool_upgrade;
use uv_cache::Cache;
use uv_configuration::Concurrency;
use uv_distribution_types::InstalledMetadata;
use uv_fs::{Simplified, CWD};
use uv_installer::compile_tree;
@ -147,6 +148,7 @@ pub(super) struct DryRunEvent<T: Display> {
/// See the `--compile` option on `pip sync` and `pip install`.
pub(super) async fn compile_bytecode(
venv: &PythonEnvironment,
concurrency: &Concurrency,
cache: &Cache,
printer: Printer,
) -> anyhow::Result<()> {
@ -154,7 +156,12 @@ pub(super) async fn compile_bytecode(
let mut files = 0;
for site_packages in venv.site_packages() {
let site_packages = CWD.join(site_packages);
files += compile_tree(&site_packages, venv.python_executable(), cache.root())
files += compile_tree(
&site_packages,
venv.python_executable(),
concurrency,
cache.root(),
)
.await
.with_context(|| {
format!(

View File

@ -554,7 +554,7 @@ pub(crate) async fn install(
}
if compile {
compile_bytecode(venv, cache, printer).await?;
compile_bytecode(venv, &concurrency, cache, printer).await?;
}
// Construct a summary of the changes made to the environment.