mirror of https://github.com/astral-sh/uv
Use install concurrency for bytecode compilation too (#11615)
Instead of always using all available threads for bytecode compilation, respect `UV_CONCURRENT_INSTALLS`, so the parallelism is configurable instead of hardcoded. We reuse the install limit since bytecode compilation only runs after install.
This commit is contained in:
parent
ab551eaf34
commit
f9b638a296
|
|
@ -4928,6 +4928,7 @@ dependencies = [
|
|||
"uv-cache",
|
||||
"uv-cli",
|
||||
"uv-client",
|
||||
"uv-configuration",
|
||||
"uv-distribution-filename",
|
||||
"uv-distribution-types",
|
||||
"uv-extract",
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ workspace = true
|
|||
uv-cache = { workspace = true, features = ["clap"] }
|
||||
uv-cli = { workspace = true }
|
||||
uv-client = { workspace = true }
|
||||
uv-configuration = { workspace = true }
|
||||
uv-distribution-filename = { workspace = true }
|
||||
uv-distribution-types = { workspace = true }
|
||||
uv-extract = { workspace = true, optional = true }
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ use clap::Parser;
|
|||
use tracing::info;
|
||||
|
||||
use uv_cache::{Cache, CacheArgs};
|
||||
use uv_configuration::Concurrency;
|
||||
use uv_python::{EnvironmentPreference, PythonEnvironment, PythonRequest};
|
||||
|
||||
#[derive(Parser)]
|
||||
|
|
@ -33,6 +34,7 @@ pub(crate) async fn compile(args: CompileArgs) -> anyhow::Result<()> {
|
|||
let files = uv_installer::compile_tree(
|
||||
&fs_err::canonicalize(args.root)?,
|
||||
&interpreter,
|
||||
&Concurrency::default(),
|
||||
cache.root(),
|
||||
)
|
||||
.await?;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
use std::num::NonZeroUsize;
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
|
|
@ -14,6 +13,7 @@ use tokio::sync::oneshot;
|
|||
use tracing::{debug, instrument};
|
||||
use walkdir::WalkDir;
|
||||
|
||||
use uv_configuration::Concurrency;
|
||||
use uv_fs::Simplified;
|
||||
use uv_static::EnvVars;
|
||||
use uv_warnings::warn_user;
|
||||
|
|
@ -71,6 +71,7 @@ pub enum CompileError {
|
|||
pub async fn compile_tree(
|
||||
dir: &Path,
|
||||
python_executable: &Path,
|
||||
concurrency: &Concurrency,
|
||||
cache: &Path,
|
||||
) -> Result<usize, CompileError> {
|
||||
debug_assert!(
|
||||
|
|
@ -78,13 +79,10 @@ pub async fn compile_tree(
|
|||
"compileall doesn't work with relative paths: `{}`",
|
||||
dir.display()
|
||||
);
|
||||
let worker_count = std::thread::available_parallelism().unwrap_or_else(|err| {
|
||||
warn_user!("Couldn't determine number of cores, compiling with a single thread: {err}");
|
||||
NonZeroUsize::MIN
|
||||
});
|
||||
let worker_count = concurrency.installs;
|
||||
|
||||
// A larger buffer is significantly faster than just 1 or the worker count.
|
||||
let (sender, receiver) = async_channel::bounded::<PathBuf>(worker_count.get() * 10);
|
||||
let (sender, receiver) = async_channel::bounded::<PathBuf>(worker_count * 10);
|
||||
|
||||
// Running Python with an actual file will produce better error messages.
|
||||
let tempdir = tempdir_in(cache).map_err(CompileError::TempFile)?;
|
||||
|
|
@ -92,7 +90,7 @@ pub async fn compile_tree(
|
|||
|
||||
debug!("Starting {} bytecode compilation workers", worker_count);
|
||||
let mut worker_handles = Vec::new();
|
||||
for _ in 0..worker_count.get() {
|
||||
for _ in 0..worker_count {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
let worker = worker(
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ pub(crate) use tool::uninstall::uninstall as tool_uninstall;
|
|||
pub(crate) use tool::update_shell::update_shell as tool_update_shell;
|
||||
pub(crate) use tool::upgrade::upgrade as tool_upgrade;
|
||||
use uv_cache::Cache;
|
||||
use uv_configuration::Concurrency;
|
||||
use uv_distribution_types::InstalledMetadata;
|
||||
use uv_fs::{Simplified, CWD};
|
||||
use uv_installer::compile_tree;
|
||||
|
|
@ -147,6 +148,7 @@ pub(super) struct DryRunEvent<T: Display> {
|
|||
/// See the `--compile` option on `pip sync` and `pip install`.
|
||||
pub(super) async fn compile_bytecode(
|
||||
venv: &PythonEnvironment,
|
||||
concurrency: &Concurrency,
|
||||
cache: &Cache,
|
||||
printer: Printer,
|
||||
) -> anyhow::Result<()> {
|
||||
|
|
@ -154,7 +156,12 @@ pub(super) async fn compile_bytecode(
|
|||
let mut files = 0;
|
||||
for site_packages in venv.site_packages() {
|
||||
let site_packages = CWD.join(site_packages);
|
||||
files += compile_tree(&site_packages, venv.python_executable(), cache.root())
|
||||
files += compile_tree(
|
||||
&site_packages,
|
||||
venv.python_executable(),
|
||||
concurrency,
|
||||
cache.root(),
|
||||
)
|
||||
.await
|
||||
.with_context(|| {
|
||||
format!(
|
||||
|
|
|
|||
|
|
@ -554,7 +554,7 @@ pub(crate) async fn install(
|
|||
}
|
||||
|
||||
if compile {
|
||||
compile_bytecode(venv, cache, printer).await?;
|
||||
compile_bytecode(venv, &concurrency, cache, printer).await?;
|
||||
}
|
||||
|
||||
// Construct a summary of the changes made to the environment.
|
||||
|
|
|
|||
Loading…
Reference in New Issue