mirror of https://github.com/astral-sh/uv
Use install concurrency for bytecode compilation too (#11615)
Instead of always using all available threads for bytecode compilation, respect `UV_CONCURRENT_INSTALLS`, so the parallelism is configurable instead of hardcoded. We reuse the install limit since bytecode compilation only runs after install.
This commit is contained in:
parent
ab551eaf34
commit
f9b638a296
|
|
@ -4928,6 +4928,7 @@ dependencies = [
|
||||||
"uv-cache",
|
"uv-cache",
|
||||||
"uv-cli",
|
"uv-cli",
|
||||||
"uv-client",
|
"uv-client",
|
||||||
|
"uv-configuration",
|
||||||
"uv-distribution-filename",
|
"uv-distribution-filename",
|
||||||
"uv-distribution-types",
|
"uv-distribution-types",
|
||||||
"uv-extract",
|
"uv-extract",
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,7 @@ workspace = true
|
||||||
uv-cache = { workspace = true, features = ["clap"] }
|
uv-cache = { workspace = true, features = ["clap"] }
|
||||||
uv-cli = { workspace = true }
|
uv-cli = { workspace = true }
|
||||||
uv-client = { workspace = true }
|
uv-client = { workspace = true }
|
||||||
|
uv-configuration = { workspace = true }
|
||||||
uv-distribution-filename = { workspace = true }
|
uv-distribution-filename = { workspace = true }
|
||||||
uv-distribution-types = { workspace = true }
|
uv-distribution-types = { workspace = true }
|
||||||
uv-extract = { workspace = true, optional = true }
|
uv-extract = { workspace = true, optional = true }
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ use clap::Parser;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
use uv_cache::{Cache, CacheArgs};
|
use uv_cache::{Cache, CacheArgs};
|
||||||
|
use uv_configuration::Concurrency;
|
||||||
use uv_python::{EnvironmentPreference, PythonEnvironment, PythonRequest};
|
use uv_python::{EnvironmentPreference, PythonEnvironment, PythonRequest};
|
||||||
|
|
||||||
#[derive(Parser)]
|
#[derive(Parser)]
|
||||||
|
|
@ -33,6 +34,7 @@ pub(crate) async fn compile(args: CompileArgs) -> anyhow::Result<()> {
|
||||||
let files = uv_installer::compile_tree(
|
let files = uv_installer::compile_tree(
|
||||||
&fs_err::canonicalize(args.root)?,
|
&fs_err::canonicalize(args.root)?,
|
||||||
&interpreter,
|
&interpreter,
|
||||||
|
&Concurrency::default(),
|
||||||
cache.root(),
|
cache.root(),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,3 @@
|
||||||
use std::num::NonZeroUsize;
|
|
||||||
use std::panic::AssertUnwindSafe;
|
use std::panic::AssertUnwindSafe;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::Stdio;
|
use std::process::Stdio;
|
||||||
|
|
@ -14,6 +13,7 @@ use tokio::sync::oneshot;
|
||||||
use tracing::{debug, instrument};
|
use tracing::{debug, instrument};
|
||||||
use walkdir::WalkDir;
|
use walkdir::WalkDir;
|
||||||
|
|
||||||
|
use uv_configuration::Concurrency;
|
||||||
use uv_fs::Simplified;
|
use uv_fs::Simplified;
|
||||||
use uv_static::EnvVars;
|
use uv_static::EnvVars;
|
||||||
use uv_warnings::warn_user;
|
use uv_warnings::warn_user;
|
||||||
|
|
@ -71,6 +71,7 @@ pub enum CompileError {
|
||||||
pub async fn compile_tree(
|
pub async fn compile_tree(
|
||||||
dir: &Path,
|
dir: &Path,
|
||||||
python_executable: &Path,
|
python_executable: &Path,
|
||||||
|
concurrency: &Concurrency,
|
||||||
cache: &Path,
|
cache: &Path,
|
||||||
) -> Result<usize, CompileError> {
|
) -> Result<usize, CompileError> {
|
||||||
debug_assert!(
|
debug_assert!(
|
||||||
|
|
@ -78,13 +79,10 @@ pub async fn compile_tree(
|
||||||
"compileall doesn't work with relative paths: `{}`",
|
"compileall doesn't work with relative paths: `{}`",
|
||||||
dir.display()
|
dir.display()
|
||||||
);
|
);
|
||||||
let worker_count = std::thread::available_parallelism().unwrap_or_else(|err| {
|
let worker_count = concurrency.installs;
|
||||||
warn_user!("Couldn't determine number of cores, compiling with a single thread: {err}");
|
|
||||||
NonZeroUsize::MIN
|
|
||||||
});
|
|
||||||
|
|
||||||
// A larger buffer is significantly faster than just 1 or the worker count.
|
// A larger buffer is significantly faster than just 1 or the worker count.
|
||||||
let (sender, receiver) = async_channel::bounded::<PathBuf>(worker_count.get() * 10);
|
let (sender, receiver) = async_channel::bounded::<PathBuf>(worker_count * 10);
|
||||||
|
|
||||||
// Running Python with an actual file will produce better error messages.
|
// Running Python with an actual file will produce better error messages.
|
||||||
let tempdir = tempdir_in(cache).map_err(CompileError::TempFile)?;
|
let tempdir = tempdir_in(cache).map_err(CompileError::TempFile)?;
|
||||||
|
|
@ -92,7 +90,7 @@ pub async fn compile_tree(
|
||||||
|
|
||||||
debug!("Starting {} bytecode compilation workers", worker_count);
|
debug!("Starting {} bytecode compilation workers", worker_count);
|
||||||
let mut worker_handles = Vec::new();
|
let mut worker_handles = Vec::new();
|
||||||
for _ in 0..worker_count.get() {
|
for _ in 0..worker_count {
|
||||||
let (tx, rx) = oneshot::channel();
|
let (tx, rx) = oneshot::channel();
|
||||||
|
|
||||||
let worker = worker(
|
let worker = worker(
|
||||||
|
|
|
||||||
|
|
@ -47,6 +47,7 @@ pub(crate) use tool::uninstall::uninstall as tool_uninstall;
|
||||||
pub(crate) use tool::update_shell::update_shell as tool_update_shell;
|
pub(crate) use tool::update_shell::update_shell as tool_update_shell;
|
||||||
pub(crate) use tool::upgrade::upgrade as tool_upgrade;
|
pub(crate) use tool::upgrade::upgrade as tool_upgrade;
|
||||||
use uv_cache::Cache;
|
use uv_cache::Cache;
|
||||||
|
use uv_configuration::Concurrency;
|
||||||
use uv_distribution_types::InstalledMetadata;
|
use uv_distribution_types::InstalledMetadata;
|
||||||
use uv_fs::{Simplified, CWD};
|
use uv_fs::{Simplified, CWD};
|
||||||
use uv_installer::compile_tree;
|
use uv_installer::compile_tree;
|
||||||
|
|
@ -147,6 +148,7 @@ pub(super) struct DryRunEvent<T: Display> {
|
||||||
/// See the `--compile` option on `pip sync` and `pip install`.
|
/// See the `--compile` option on `pip sync` and `pip install`.
|
||||||
pub(super) async fn compile_bytecode(
|
pub(super) async fn compile_bytecode(
|
||||||
venv: &PythonEnvironment,
|
venv: &PythonEnvironment,
|
||||||
|
concurrency: &Concurrency,
|
||||||
cache: &Cache,
|
cache: &Cache,
|
||||||
printer: Printer,
|
printer: Printer,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
|
|
@ -154,14 +156,19 @@ pub(super) async fn compile_bytecode(
|
||||||
let mut files = 0;
|
let mut files = 0;
|
||||||
for site_packages in venv.site_packages() {
|
for site_packages in venv.site_packages() {
|
||||||
let site_packages = CWD.join(site_packages);
|
let site_packages = CWD.join(site_packages);
|
||||||
files += compile_tree(&site_packages, venv.python_executable(), cache.root())
|
files += compile_tree(
|
||||||
.await
|
&site_packages,
|
||||||
.with_context(|| {
|
venv.python_executable(),
|
||||||
format!(
|
concurrency,
|
||||||
"Failed to bytecode-compile Python file in: {}",
|
cache.root(),
|
||||||
site_packages.user_display()
|
)
|
||||||
)
|
.await
|
||||||
})?;
|
.with_context(|| {
|
||||||
|
format!(
|
||||||
|
"Failed to bytecode-compile Python file in: {}",
|
||||||
|
site_packages.user_display()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
}
|
}
|
||||||
let s = if files == 1 { "" } else { "s" };
|
let s = if files == 1 { "" } else { "s" };
|
||||||
writeln!(
|
writeln!(
|
||||||
|
|
|
||||||
|
|
@ -554,7 +554,7 @@ pub(crate) async fn install(
|
||||||
}
|
}
|
||||||
|
|
||||||
if compile {
|
if compile {
|
||||||
compile_bytecode(venv, cache, printer).await?;
|
compile_bytecode(venv, &concurrency, cache, printer).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Construct a summary of the changes made to the environment.
|
// Construct a summary of the changes made to the environment.
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue