From d6738679197fd5b40754d62e84a7d3d625e60206 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 15 Jan 2026 12:17:46 -0600 Subject: [PATCH] Adjust the process ulimit to the maximum allowed on startup (#17464) Closes #16999 See the commentary at https://github.com/astral-sh/uv/issues/16999#issuecomment-3641417484 for precedence in other tools. This is non-fatal on error, but there are various scary behaviors from increased ulimit sizes so I've gated this with preview. --------- Co-authored-by: Claude --- Cargo.lock | 9 +++ Cargo.toml | 3 +- crates/uv-preview/src/lib.rs | 7 ++ crates/uv-unix/Cargo.toml | 20 +++++ crates/uv-unix/src/lib.rs | 9 +++ crates/uv-unix/src/resource_limits.rs | 111 ++++++++++++++++++++++++++ crates/uv/Cargo.toml | 1 + crates/uv/src/lib.rs | 11 +++ crates/uv/tests/it/show_settings.rs | 4 +- 9 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 crates/uv-unix/Cargo.toml create mode 100644 crates/uv-unix/src/lib.rs create mode 100644 crates/uv-unix/src/resource_limits.rs diff --git a/Cargo.lock b/Cargo.lock index ce8ade5ac..4cd1b3889 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5707,6 +5707,7 @@ dependencies = [ "uv-torch", "uv-trampoline-builder", "uv-types", + "uv-unix", "uv-version", "uv-virtualenv", "uv-warnings", @@ -7136,6 +7137,14 @@ dependencies = [ "uv-workspace", ] +[[package]] +name = "uv-unix" +version = "0.0.14" +dependencies = [ + "nix", + "thiserror 2.0.17", +] + [[package]] name = "uv-version" version = "0.9.25" diff --git a/Cargo.toml b/Cargo.toml index a8e96b278..ef85d4cb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,7 @@ uv-tool = { version = "0.0.14", path = "crates/uv-tool" } uv-torch = { version = "0.0.14", path = "crates/uv-torch" } uv-trampoline-builder = { version = "0.0.14", path = "crates/uv-trampoline-builder" } uv-types = { version = "0.0.14", path = "crates/uv-types" } +uv-unix = { version = "0.0.14", path = "crates/uv-unix" } uv-version = { version = "0.9.25", path = "crates/uv-version" } uv-virtualenv = { version = "0.0.14", path = "crates/uv-virtualenv" } uv-warnings = { version = "0.0.14", path = "crates/uv-warnings" } @@ -134,7 +135,7 @@ md-5 = { version = "0.10.6" } memchr = { version = "2.7.4" } miette = { version = "7.2.0", features = ["fancy-no-backtrace"] } nanoid = { version = "0.4.0" } -nix = { version = "0.30.0", features = ["signal"] } +nix = { version = "0.30.0", features = ["resource", "signal"] } open = { version = "5.3.2" } owo-colors = { version = "4.1.0" } path-slash = { version = "0.2.1" } diff --git a/crates/uv-preview/src/lib.rs b/crates/uv-preview/src/lib.rs index e2acd6adf..3bf01778b 100644 --- a/crates/uv-preview/src/lib.rs +++ b/crates/uv-preview/src/lib.rs @@ -31,6 +31,7 @@ bitflags::bitflags! { const TARGET_WORKSPACE_DISCOVERY = 1 << 19; const METADATA_JSON = 1 << 20; const GCS_ENDPOINT = 1 << 21; + const ADJUST_ULIMIT = 1 << 22; } } @@ -62,6 +63,7 @@ impl PreviewFeatures { Self::TARGET_WORKSPACE_DISCOVERY => "target-workspace-discovery", Self::METADATA_JSON => "metadata-json", Self::GCS_ENDPOINT => "gcs-endpoint", + Self::ADJUST_ULIMIT => "adjust-ulimit", _ => panic!("`flag_as_str` can only be used for exactly one feature flag"), } } @@ -121,6 +123,7 @@ impl FromStr for PreviewFeatures { "direct-publish" => Self::DIRECT_PUBLISH, "target-workspace-discovery" => Self::TARGET_WORKSPACE_DISCOVERY, "metadata-json" => Self::METADATA_JSON, + "adjust-ulimit" => Self::ADJUST_ULIMIT, _ => { warn_user_once!("Unknown preview feature: `{part}`"); continue; @@ -310,6 +313,10 @@ mod tests { PreviewFeatures::METADATA_JSON.flag_as_str(), "metadata-json" ); + assert_eq!( + PreviewFeatures::ADJUST_ULIMIT.flag_as_str(), + "adjust-ulimit" + ); } #[test] diff --git a/crates/uv-unix/Cargo.toml b/crates/uv-unix/Cargo.toml new file mode 100644 index 000000000..7ea11f363 --- /dev/null +++ b/crates/uv-unix/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "uv-unix" +version = "0.0.14" +description = "Unix-specific functionality for uv" +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[target.'cfg(unix)'.dependencies] +nix = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/uv-unix/src/lib.rs b/crates/uv-unix/src/lib.rs new file mode 100644 index 000000000..e276391a8 --- /dev/null +++ b/crates/uv-unix/src/lib.rs @@ -0,0 +1,9 @@ +//! Unix-specific functionality for uv. +//! +//! This crate is only functional on Unix platforms. + +#![cfg(unix)] + +mod resource_limits; + +pub use resource_limits::{OpenFileLimitError, adjust_open_file_limit}; diff --git a/crates/uv-unix/src/resource_limits.rs b/crates/uv-unix/src/resource_limits.rs new file mode 100644 index 000000000..b28729964 --- /dev/null +++ b/crates/uv-unix/src/resource_limits.rs @@ -0,0 +1,111 @@ +//! Helper for adjusting Unix resource limits. +//! +//! Linux has a historically low default limit of 1024 open file descriptors per process. +//! macOS also defaults to a low soft limit (typically 256), though its hard limit is much +//! higher. On modern multi-core machines, these low defaults can cause "too many open files" +//! errors because uv infers concurrency limits from CPU count and may schedule more concurrent +//! work than the default file descriptor limit allows. +//! +//! This module attempts to raise the soft limit to the hard limit at startup to avoid these +//! errors without requiring users to manually configure their shell's `ulimit` settings. +//! The raised limit is inherited by child processes, which is important for commands like +//! `uv run` that spawn Python interpreters. +//! +//! See: + +use nix::errno::Errno; +use nix::sys::resource::{Resource, getrlimit, rlim_t, setrlimit}; +use thiserror::Error; + +/// Errors that can occur when adjusting resource limits. +#[derive(Debug, Error)] +pub enum OpenFileLimitError { + #[error("failed to get open file limit: {}", .0.desc())] + GetLimitFailed(Errno), + + #[error("encountered unexpected negative soft limit: {value}")] + NegativeSoftLimit { value: rlim_t }, + + #[error("soft limit ({current}) already meets the target ({target})")] + AlreadySufficient { current: u64, target: u64 }, + + #[error("failed to raise open file limit from {current} to {target}: {}", source.desc())] + SetLimitFailed { + current: u64, + target: u64, + source: Errno, + }, +} + +/// Maximum file descriptor limit to request. +/// +/// We cap at 0x100000 (1,048,576) to match the typical Linux default (`/proc/sys/fs/nr_open`) +/// and to avoid issues with extremely high limits. +/// +/// `OpenJDK` uses this same cap because: +/// +/// 1. Some code breaks if `RLIMIT_NOFILE` exceeds `i32::MAX` (despite the type being `u64`) +/// 2. Code that iterates over all possible FDs, e.g., to close them, can timeout +/// +/// See: +/// See: +/// +/// Note: `rlim_t` is platform-specific (`u64` on Linux/macOS, `i64` on FreeBSD). +const MAX_NOFILE_LIMIT: rlim_t = 0x0010_0000; + +/// Attempt to raise the open file descriptor limit to the maximum allowed. +/// +/// This function tries to set the soft limit to `min(hard_limit, 0x100000)`. If the operation +/// fails, it returns an error since the default limits may still be sufficient for the +/// current workload. +/// +/// Returns [`Ok`] with the new soft limit on successful adjustment, or an appropriate +/// [`OpenFileLimitError`] if adjustment failed. +/// +/// Note the type of `rlim_t` is platform-specific (`u64` on Linux/macOS, `i64` on FreeBSD), but +/// this function always returns a [`u64`]. +pub fn adjust_open_file_limit() -> Result { + let (soft, hard) = + getrlimit(Resource::RLIMIT_NOFILE).map_err(OpenFileLimitError::GetLimitFailed)?; + + // Convert `rlim_t` to `u64`. On FreeBSD, `rlim_t` is `i64` which may fail. + // On Linux and macOS, `rlim_t` is a `u64`, and the conversion is infallible. + let Some(soft) = rlim_t_to_u64(soft) else { + return Err(OpenFileLimitError::NegativeSoftLimit { value: soft }); + }; + + // Cap the target limit to avoid issues with extremely high values. + // If hard is negative or exceeds MAX_NOFILE_LIMIT, use MAX_NOFILE_LIMIT. + #[allow(clippy::unnecessary_cast)] + let target = rlim_t_to_u64(hard.min(MAX_NOFILE_LIMIT)).unwrap_or(MAX_NOFILE_LIMIT as u64); + + if soft >= target { + return Err(OpenFileLimitError::AlreadySufficient { + current: soft, + target, + }); + } + + // Try to raise the soft limit to the target. + // Safe because target <= MAX_NOFILE_LIMIT which fits in both i64 and u64. + let target_rlim = target as rlim_t; + + setrlimit(Resource::RLIMIT_NOFILE, target_rlim, hard).map_err(|err| { + OpenFileLimitError::SetLimitFailed { + current: soft, + target, + source: err, + } + })?; + + Ok(target) +} + +/// Convert `rlim_t` to `u64`, returning `None` if negative. +/// +/// On Linux/macOS, `rlim_t` is `u64` so this always succeeds. +/// On FreeBSD, `rlim_t` is `i64` so negative values return `None`. +#[allow(clippy::unnecessary_cast, clippy::useless_conversion)] +fn rlim_t_to_u64(value: rlim_t) -> Option { + u64::try_from(value).ok() +} diff --git a/crates/uv/Cargo.toml b/crates/uv/Cargo.toml index 6c9a0e627..7769033e0 100644 --- a/crates/uv/Cargo.toml +++ b/crates/uv/Cargo.toml @@ -149,6 +149,7 @@ zip = { workspace = true } [target.'cfg(unix)'.dependencies] nix = { workspace = true } +uv-unix = { workspace = true } [features] default = ["performance", "uv-distribution/static", "default-tests"] diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index 145380799..3560faa27 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -338,6 +338,17 @@ async fn run(mut cli: Cli) -> Result { &environment, ); + // Adjust open file limits on Unix if the preview feature is enabled. + #[cfg(unix)] + if globals.preview.is_enabled(PreviewFeatures::ADJUST_ULIMIT) { + match uv_unix::adjust_open_file_limit() { + Ok(_) | Err(uv_unix::OpenFileLimitError::AlreadySufficient { .. }) => {} + // TODO(zanieb): When moving out of preview, consider changing this to a log instead of + // a warning because it's okay if we fail here. + Err(err) => warn_user!("{err}"), + } + } + // Resolve the cache settings. let cache_settings = CacheSettings::resolve(*cli.top_level.cache_args, filesystem.as_ref()); diff --git a/crates/uv/tests/it/show_settings.rs b/crates/uv/tests/it/show_settings.rs index dc304fc43..88327c3a5 100644 --- a/crates/uv/tests/it/show_settings.rs +++ b/crates/uv/tests/it/show_settings.rs @@ -7982,7 +7982,7 @@ fn preview_features() { show_settings: true, preview: Preview { flags: PreviewFeatures( - PYTHON_INSTALL_DEFAULT | PYTHON_UPGRADE | JSON_OUTPUT | PYLOCK | ADD_BOUNDS | PACKAGE_CONFLICTS | EXTRA_BUILD_DEPENDENCIES | DETECT_MODULE_CONFLICTS | FORMAT | NATIVE_AUTH | S3_ENDPOINT | CACHE_SIZE | INIT_PROJECT_FLAG | WORKSPACE_METADATA | WORKSPACE_DIR | WORKSPACE_LIST | SBOM_EXPORT | AUTH_HELPER | DIRECT_PUBLISH | TARGET_WORKSPACE_DISCOVERY | METADATA_JSON | GCS_ENDPOINT, + PYTHON_INSTALL_DEFAULT | PYTHON_UPGRADE | JSON_OUTPUT | PYLOCK | ADD_BOUNDS | PACKAGE_CONFLICTS | EXTRA_BUILD_DEPENDENCIES | DETECT_MODULE_CONFLICTS | FORMAT | NATIVE_AUTH | S3_ENDPOINT | CACHE_SIZE | INIT_PROJECT_FLAG | WORKSPACE_METADATA | WORKSPACE_DIR | WORKSPACE_LIST | SBOM_EXPORT | AUTH_HELPER | DIRECT_PUBLISH | TARGET_WORKSPACE_DISCOVERY | METADATA_JSON | GCS_ENDPOINT | ADJUST_ULIMIT, ), }, python_preference: Managed, @@ -8220,7 +8220,7 @@ fn preview_features() { show_settings: true, preview: Preview { flags: PreviewFeatures( - PYTHON_INSTALL_DEFAULT | PYTHON_UPGRADE | JSON_OUTPUT | PYLOCK | ADD_BOUNDS | PACKAGE_CONFLICTS | EXTRA_BUILD_DEPENDENCIES | DETECT_MODULE_CONFLICTS | FORMAT | NATIVE_AUTH | S3_ENDPOINT | CACHE_SIZE | INIT_PROJECT_FLAG | WORKSPACE_METADATA | WORKSPACE_DIR | WORKSPACE_LIST | SBOM_EXPORT | AUTH_HELPER | DIRECT_PUBLISH | TARGET_WORKSPACE_DISCOVERY | METADATA_JSON | GCS_ENDPOINT, + PYTHON_INSTALL_DEFAULT | PYTHON_UPGRADE | JSON_OUTPUT | PYLOCK | ADD_BOUNDS | PACKAGE_CONFLICTS | EXTRA_BUILD_DEPENDENCIES | DETECT_MODULE_CONFLICTS | FORMAT | NATIVE_AUTH | S3_ENDPOINT | CACHE_SIZE | INIT_PROJECT_FLAG | WORKSPACE_METADATA | WORKSPACE_DIR | WORKSPACE_LIST | SBOM_EXPORT | AUTH_HELPER | DIRECT_PUBLISH | TARGET_WORKSPACE_DISCOVERY | METADATA_JSON | GCS_ENDPOINT | ADJUST_ULIMIT, ), }, python_preference: Managed,